2024-12-12 06:12:25,424 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-12-12 06:12:25,437 main DEBUG Took 0.011116 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-12 06:12:25,437 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-12 06:12:25,437 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-12 06:12:25,438 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-12 06:12:25,439 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-12 06:12:25,446 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-12 06:12:25,460 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-12 06:12:25,461 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-12 06:12:25,462 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-12 06:12:25,462 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-12 06:12:25,463 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-12 06:12:25,464 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-12 06:12:25,465 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-12 06:12:25,465 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-12 06:12:25,466 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-12 06:12:25,466 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-12 06:12:25,467 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-12 06:12:25,467 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-12 06:12:25,468 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-12 06:12:25,468 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-12 06:12:25,469 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-12 06:12:25,469 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-12 06:12:25,469 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-12 06:12:25,470 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-12 06:12:25,470 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-12 06:12:25,471 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-12 06:12:25,471 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-12 06:12:25,472 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-12 06:12:25,472 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-12 06:12:25,472 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-12 06:12:25,473 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-12 06:12:25,473 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-12 06:12:25,475 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-12 06:12:25,477 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-12 06:12:25,479 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-12 06:12:25,480 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-12 06:12:25,482 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-12 06:12:25,482 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-12 06:12:25,490 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-12 06:12:25,493 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-12 06:12:25,494 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-12 06:12:25,495 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-12 06:12:25,495 main DEBUG createAppenders(={Console}) 2024-12-12 06:12:25,496 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 initialized 2024-12-12 06:12:25,496 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-12-12 06:12:25,497 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 OK. 2024-12-12 06:12:25,497 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-12 06:12:25,498 main DEBUG OutputStream closed 2024-12-12 06:12:25,498 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-12 06:12:25,498 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-12 06:12:25,499 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@c88a337 OK 2024-12-12 06:12:25,578 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-12 06:12:25,581 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-12 06:12:25,582 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-12 06:12:25,583 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-12 06:12:25,584 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-12 06:12:25,584 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-12 06:12:25,585 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-12 06:12:25,585 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-12 06:12:25,585 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-12 06:12:25,586 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-12 06:12:25,586 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-12 06:12:25,587 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-12 06:12:25,587 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-12 06:12:25,588 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-12 06:12:25,588 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-12 06:12:25,588 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-12 06:12:25,588 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-12 06:12:25,589 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-12 06:12:25,591 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-12 06:12:25,591 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@13330ac6) with optional ClassLoader: null 2024-12-12 06:12:25,592 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-12 06:12:25,592 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@13330ac6] started OK. 2024-12-12T06:12:25,604 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP timeout: 13 mins 2024-12-12 06:12:25,606 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-12 06:12:25,607 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-12T06:12:25,827 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/81c28be5-9416-b8f7-f79e-42c4fbc92d9e 2024-12-12T06:12:25,842 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$HMasterForTest, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-12T06:12:25,857 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/81c28be5-9416-b8f7-f79e-42c4fbc92d9e/cluster_3ed8416f-f61b-db52-c71b-dadd84a95366, deleteOnExit=true 2024-12-12T06:12:25,857 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-12T06:12:25,858 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/81c28be5-9416-b8f7-f79e-42c4fbc92d9e/test.cache.data in system properties and HBase conf 2024-12-12T06:12:25,858 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/81c28be5-9416-b8f7-f79e-42c4fbc92d9e/hadoop.tmp.dir in system properties and HBase conf 2024-12-12T06:12:25,859 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/81c28be5-9416-b8f7-f79e-42c4fbc92d9e/hadoop.log.dir in system properties and HBase conf 2024-12-12T06:12:25,859 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/81c28be5-9416-b8f7-f79e-42c4fbc92d9e/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-12T06:12:25,860 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/81c28be5-9416-b8f7-f79e-42c4fbc92d9e/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-12T06:12:25,860 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-12T06:12:25,942 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-12T06:12:26,028 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-12T06:12:26,031 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/81c28be5-9416-b8f7-f79e-42c4fbc92d9e/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-12T06:12:26,032 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/81c28be5-9416-b8f7-f79e-42c4fbc92d9e/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-12T06:12:26,032 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/81c28be5-9416-b8f7-f79e-42c4fbc92d9e/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-12T06:12:26,032 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/81c28be5-9416-b8f7-f79e-42c4fbc92d9e/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-12T06:12:26,033 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/81c28be5-9416-b8f7-f79e-42c4fbc92d9e/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-12T06:12:26,033 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/81c28be5-9416-b8f7-f79e-42c4fbc92d9e/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-12T06:12:26,034 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/81c28be5-9416-b8f7-f79e-42c4fbc92d9e/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-12T06:12:26,034 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/81c28be5-9416-b8f7-f79e-42c4fbc92d9e/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-12T06:12:26,035 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/81c28be5-9416-b8f7-f79e-42c4fbc92d9e/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-12T06:12:26,035 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/81c28be5-9416-b8f7-f79e-42c4fbc92d9e/nfs.dump.dir in system properties and HBase conf 2024-12-12T06:12:26,035 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/81c28be5-9416-b8f7-f79e-42c4fbc92d9e/java.io.tmpdir in system properties and HBase conf 2024-12-12T06:12:26,036 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/81c28be5-9416-b8f7-f79e-42c4fbc92d9e/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-12T06:12:26,036 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/81c28be5-9416-b8f7-f79e-42c4fbc92d9e/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-12T06:12:26,036 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/81c28be5-9416-b8f7-f79e-42c4fbc92d9e/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-12T06:12:26,943 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-12T06:12:27,005 INFO [Time-limited test {}] log.Log(170): Logging initialized @2165ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-12T06:12:27,066 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-12T06:12:27,117 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-12T06:12:27,135 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-12T06:12:27,136 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-12T06:12:27,137 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-12T06:12:27,147 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-12T06:12:27,150 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2558e43b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/81c28be5-9416-b8f7-f79e-42c4fbc92d9e/hadoop.log.dir/,AVAILABLE} 2024-12-12T06:12:27,152 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@42f36157{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-12T06:12:27,309 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7a5b8304{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/81c28be5-9416-b8f7-f79e-42c4fbc92d9e/java.io.tmpdir/jetty-localhost-33379-hadoop-hdfs-3_4_1-tests_jar-_-any-17009790658199800041/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-12T06:12:27,315 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@63f7414d{HTTP/1.1, (http/1.1)}{localhost:33379} 2024-12-12T06:12:27,315 INFO [Time-limited test {}] server.Server(415): Started @2476ms 2024-12-12T06:12:27,804 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-12T06:12:27,813 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-12T06:12:27,814 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-12T06:12:27,814 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-12T06:12:27,814 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-12T06:12:27,815 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@325049b2{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/81c28be5-9416-b8f7-f79e-42c4fbc92d9e/hadoop.log.dir/,AVAILABLE} 2024-12-12T06:12:27,816 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@44ebf1c5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-12T06:12:27,925 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2052a170{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/81c28be5-9416-b8f7-f79e-42c4fbc92d9e/java.io.tmpdir/jetty-localhost-43331-hadoop-hdfs-3_4_1-tests_jar-_-any-9096797742963905394/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-12T06:12:27,925 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@38862d81{HTTP/1.1, (http/1.1)}{localhost:43331} 2024-12-12T06:12:27,926 INFO [Time-limited test {}] server.Server(415): Started @3087ms 2024-12-12T06:12:27,974 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-12T06:12:28,071 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-12T06:12:28,076 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-12T06:12:28,077 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-12T06:12:28,077 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-12T06:12:28,077 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-12T06:12:28,078 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@35d8c8c1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/81c28be5-9416-b8f7-f79e-42c4fbc92d9e/hadoop.log.dir/,AVAILABLE} 2024-12-12T06:12:28,078 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@fe80121{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-12T06:12:28,192 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@66eb784a{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/81c28be5-9416-b8f7-f79e-42c4fbc92d9e/java.io.tmpdir/jetty-localhost-40605-hadoop-hdfs-3_4_1-tests_jar-_-any-17692308353745940628/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-12T06:12:28,192 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5f1d2dc{HTTP/1.1, (http/1.1)}{localhost:40605} 2024-12-12T06:12:28,192 INFO [Time-limited test {}] server.Server(415): Started @3354ms 2024-12-12T06:12:28,195 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-12T06:12:28,225 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-12T06:12:28,230 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-12T06:12:28,232 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-12T06:12:28,232 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-12T06:12:28,232 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-12T06:12:28,233 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7f6091c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/81c28be5-9416-b8f7-f79e-42c4fbc92d9e/hadoop.log.dir/,AVAILABLE} 2024-12-12T06:12:28,234 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@423dea5a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-12T06:12:28,326 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@19dc7bb{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/81c28be5-9416-b8f7-f79e-42c4fbc92d9e/java.io.tmpdir/jetty-localhost-44695-hadoop-hdfs-3_4_1-tests_jar-_-any-18304887673999016203/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-12T06:12:28,326 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3b41df7e{HTTP/1.1, (http/1.1)}{localhost:44695} 2024-12-12T06:12:28,327 INFO [Time-limited test {}] server.Server(415): Started @3488ms 2024-12-12T06:12:28,328 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-12T06:12:29,292 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/81c28be5-9416-b8f7-f79e-42c4fbc92d9e/cluster_3ed8416f-f61b-db52-c71b-dadd84a95366/data/data1/current/BP-548786712-172.17.0.2-1733983946502/current, will proceed with Du for space computation calculation, 2024-12-12T06:12:29,292 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/81c28be5-9416-b8f7-f79e-42c4fbc92d9e/cluster_3ed8416f-f61b-db52-c71b-dadd84a95366/data/data2/current/BP-548786712-172.17.0.2-1733983946502/current, will proceed with Du for space computation calculation, 2024-12-12T06:12:29,292 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/81c28be5-9416-b8f7-f79e-42c4fbc92d9e/cluster_3ed8416f-f61b-db52-c71b-dadd84a95366/data/data3/current/BP-548786712-172.17.0.2-1733983946502/current, will proceed with Du for space computation calculation, 2024-12-12T06:12:29,292 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/81c28be5-9416-b8f7-f79e-42c4fbc92d9e/cluster_3ed8416f-f61b-db52-c71b-dadd84a95366/data/data4/current/BP-548786712-172.17.0.2-1733983946502/current, will proceed with Du for space computation calculation, 2024-12-12T06:12:29,321 WARN [Thread-136 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/81c28be5-9416-b8f7-f79e-42c4fbc92d9e/cluster_3ed8416f-f61b-db52-c71b-dadd84a95366/data/data5/current/BP-548786712-172.17.0.2-1733983946502/current, will proceed with Du for space computation calculation, 2024-12-12T06:12:29,321 WARN [Thread-137 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/81c28be5-9416-b8f7-f79e-42c4fbc92d9e/cluster_3ed8416f-f61b-db52-c71b-dadd84a95366/data/data6/current/BP-548786712-172.17.0.2-1733983946502/current, will proceed with Du for space computation calculation, 2024-12-12T06:12:29,327 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-12T06:12:29,327 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-12T06:12:29,347 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-12T06:12:29,387 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc6ada79a55715ab2 with lease ID 0xa5169ef870b8c924: Processing first storage report for DS-2e86db76-0bd9-4ac5-b9f1-045da95823f2 from datanode DatanodeRegistration(127.0.0.1:38541, datanodeUuid=ad24e0e8-9462-4b82-ba42-211260e601aa, infoPort=39173, infoSecurePort=0, ipcPort=41463, storageInfo=lv=-57;cid=testClusterID;nsid=1378130010;c=1733983946502) 2024-12-12T06:12:29,389 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc6ada79a55715ab2 with lease ID 0xa5169ef870b8c924: from storage DS-2e86db76-0bd9-4ac5-b9f1-045da95823f2 node DatanodeRegistration(127.0.0.1:38541, datanodeUuid=ad24e0e8-9462-4b82-ba42-211260e601aa, infoPort=39173, infoSecurePort=0, ipcPort=41463, storageInfo=lv=-57;cid=testClusterID;nsid=1378130010;c=1733983946502), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-12-12T06:12:29,389 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3fea6646fa06216c with lease ID 0xa5169ef870b8c923: Processing first storage report for DS-bf6839c4-19ca-45ad-a024-f2d33b255052 from datanode DatanodeRegistration(127.0.0.1:43493, datanodeUuid=9c993caf-b2d8-4cdb-91d8-057921193aa5, infoPort=42655, infoSecurePort=0, ipcPort=39587, storageInfo=lv=-57;cid=testClusterID;nsid=1378130010;c=1733983946502) 2024-12-12T06:12:29,389 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3fea6646fa06216c with lease ID 0xa5169ef870b8c923: from storage DS-bf6839c4-19ca-45ad-a024-f2d33b255052 node DatanodeRegistration(127.0.0.1:43493, datanodeUuid=9c993caf-b2d8-4cdb-91d8-057921193aa5, infoPort=42655, infoSecurePort=0, ipcPort=39587, storageInfo=lv=-57;cid=testClusterID;nsid=1378130010;c=1733983946502), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-12T06:12:29,390 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7e28a94709b57ae2 with lease ID 0xa5169ef870b8c922: Processing first storage report for DS-6cdf379f-c6b3-472d-9e31-f23a652ce941 from datanode DatanodeRegistration(127.0.0.1:42285, datanodeUuid=780c3b0e-ff1a-43e7-9b7f-2451baf23f0d, infoPort=44435, infoSecurePort=0, ipcPort=41041, storageInfo=lv=-57;cid=testClusterID;nsid=1378130010;c=1733983946502) 2024-12-12T06:12:29,390 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7e28a94709b57ae2 with lease ID 0xa5169ef870b8c922: from storage DS-6cdf379f-c6b3-472d-9e31-f23a652ce941 node DatanodeRegistration(127.0.0.1:42285, datanodeUuid=780c3b0e-ff1a-43e7-9b7f-2451baf23f0d, infoPort=44435, infoSecurePort=0, ipcPort=41041, storageInfo=lv=-57;cid=testClusterID;nsid=1378130010;c=1733983946502), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-12T06:12:29,390 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc6ada79a55715ab2 with lease ID 0xa5169ef870b8c924: Processing first storage report for DS-33257b9f-1fe0-4280-83a8-407e13c9879b from datanode DatanodeRegistration(127.0.0.1:38541, datanodeUuid=ad24e0e8-9462-4b82-ba42-211260e601aa, infoPort=39173, infoSecurePort=0, ipcPort=41463, storageInfo=lv=-57;cid=testClusterID;nsid=1378130010;c=1733983946502) 2024-12-12T06:12:29,390 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc6ada79a55715ab2 with lease ID 0xa5169ef870b8c924: from storage DS-33257b9f-1fe0-4280-83a8-407e13c9879b node DatanodeRegistration(127.0.0.1:38541, datanodeUuid=ad24e0e8-9462-4b82-ba42-211260e601aa, infoPort=39173, infoSecurePort=0, ipcPort=41463, storageInfo=lv=-57;cid=testClusterID;nsid=1378130010;c=1733983946502), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-12T06:12:29,390 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3fea6646fa06216c with lease ID 0xa5169ef870b8c923: Processing first storage report for DS-e7f1a692-828e-474d-baa0-a1fa5b126f1a from datanode DatanodeRegistration(127.0.0.1:43493, datanodeUuid=9c993caf-b2d8-4cdb-91d8-057921193aa5, infoPort=42655, infoSecurePort=0, ipcPort=39587, storageInfo=lv=-57;cid=testClusterID;nsid=1378130010;c=1733983946502) 2024-12-12T06:12:29,390 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3fea6646fa06216c with lease ID 0xa5169ef870b8c923: from storage DS-e7f1a692-828e-474d-baa0-a1fa5b126f1a node DatanodeRegistration(127.0.0.1:43493, datanodeUuid=9c993caf-b2d8-4cdb-91d8-057921193aa5, infoPort=42655, infoSecurePort=0, ipcPort=39587, storageInfo=lv=-57;cid=testClusterID;nsid=1378130010;c=1733983946502), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-12T06:12:29,391 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7e28a94709b57ae2 with lease ID 0xa5169ef870b8c922: Processing first storage report for DS-9ad130dc-b9b3-4702-bb5c-32de94de6312 from datanode DatanodeRegistration(127.0.0.1:42285, datanodeUuid=780c3b0e-ff1a-43e7-9b7f-2451baf23f0d, infoPort=44435, infoSecurePort=0, ipcPort=41041, storageInfo=lv=-57;cid=testClusterID;nsid=1378130010;c=1733983946502) 2024-12-12T06:12:29,391 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7e28a94709b57ae2 with lease ID 0xa5169ef870b8c922: from storage DS-9ad130dc-b9b3-4702-bb5c-32de94de6312 node DatanodeRegistration(127.0.0.1:42285, datanodeUuid=780c3b0e-ff1a-43e7-9b7f-2451baf23f0d, infoPort=44435, infoSecurePort=0, ipcPort=41041, storageInfo=lv=-57;cid=testClusterID;nsid=1378130010;c=1733983946502), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-12T06:12:29,414 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/81c28be5-9416-b8f7-f79e-42c4fbc92d9e 2024-12-12T06:12:29,479 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/81c28be5-9416-b8f7-f79e-42c4fbc92d9e/cluster_3ed8416f-f61b-db52-c71b-dadd84a95366/zookeeper_0, clientPort=59802, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/81c28be5-9416-b8f7-f79e-42c4fbc92d9e/cluster_3ed8416f-f61b-db52-c71b-dadd84a95366/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/81c28be5-9416-b8f7-f79e-42c4fbc92d9e/cluster_3ed8416f-f61b-db52-c71b-dadd84a95366/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-12T06:12:29,488 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59802 2024-12-12T06:12:29,501 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-12T06:12:29,506 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-12T06:12:29,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42285 is added to blk_1073741825_1001 (size=7) 2024-12-12T06:12:29,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38541 is added to blk_1073741825_1001 (size=7) 2024-12-12T06:12:29,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43493 is added to blk_1073741825_1001 (size=7) 2024-12-12T06:12:30,151 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0 with version=8 2024-12-12T06:12:30,151 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/hbase-staging 2024-12-12T06:12:30,227 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-12T06:12:30,482 INFO [Time-limited test {}] client.ConnectionUtils(128): master/ffbfd3107920:0 server-side Connection retries=45 2024-12-12T06:12:30,491 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-12T06:12:30,491 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-12T06:12:30,495 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-12T06:12:30,496 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-12T06:12:30,496 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-12T06:12:30,615 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-12T06:12:30,667 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-12T06:12:30,675 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-12T06:12:30,679 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-12T06:12:30,701 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 105325 (auto-detected) 2024-12-12T06:12:30,702 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-12T06:12:30,719 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45209 2024-12-12T06:12:30,736 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:45209 connecting to ZooKeeper ensemble=127.0.0.1:59802 2024-12-12T06:12:30,829 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:452090x0, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-12T06:12:30,833 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:45209-0x10018b511340000 connected 2024-12-12T06:12:30,938 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-12T06:12:30,944 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-12T06:12:30,955 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45209-0x10018b511340000, quorum=127.0.0.1:59802, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-12T06:12:30,959 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0, hbase.cluster.distributed=false 2024-12-12T06:12:30,980 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45209-0x10018b511340000, quorum=127.0.0.1:59802, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-12T06:12:30,985 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45209 2024-12-12T06:12:30,985 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45209 2024-12-12T06:12:30,986 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45209 2024-12-12T06:12:30,987 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45209 2024-12-12T06:12:30,991 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45209 2024-12-12T06:12:31,078 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/ffbfd3107920:0 server-side Connection retries=45 2024-12-12T06:12:31,080 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-12T06:12:31,080 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-12T06:12:31,080 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-12T06:12:31,080 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-12T06:12:31,080 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-12T06:12:31,083 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-12T06:12:31,084 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-12T06:12:31,085 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46199 2024-12-12T06:12:31,087 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46199 connecting to ZooKeeper ensemble=127.0.0.1:59802 2024-12-12T06:12:31,088 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-12T06:12:31,091 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-12T06:12:31,111 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:461990x0, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-12T06:12:31,112 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:461990x0, quorum=127.0.0.1:59802, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-12T06:12:31,112 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46199-0x10018b511340001 connected 2024-12-12T06:12:31,118 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-12T06:12:31,127 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-12T06:12:31,130 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46199-0x10018b511340001, quorum=127.0.0.1:59802, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-12T06:12:31,135 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46199-0x10018b511340001, quorum=127.0.0.1:59802, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-12T06:12:31,135 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46199 2024-12-12T06:12:31,136 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46199 2024-12-12T06:12:31,136 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46199 2024-12-12T06:12:31,137 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46199 2024-12-12T06:12:31,137 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46199 2024-12-12T06:12:31,152 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/ffbfd3107920:0 server-side Connection retries=45 2024-12-12T06:12:31,152 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-12T06:12:31,152 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-12T06:12:31,153 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-12T06:12:31,153 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-12T06:12:31,153 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-12T06:12:31,153 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-12T06:12:31,153 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-12T06:12:31,154 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33455 2024-12-12T06:12:31,156 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33455 connecting to ZooKeeper ensemble=127.0.0.1:59802 2024-12-12T06:12:31,157 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-12T06:12:31,162 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-12T06:12:31,194 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:334550x0, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-12T06:12:31,195 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33455-0x10018b511340002 connected 2024-12-12T06:12:31,196 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33455-0x10018b511340002, quorum=127.0.0.1:59802, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-12T06:12:31,196 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-12T06:12:31,197 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-12T06:12:31,199 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33455-0x10018b511340002, quorum=127.0.0.1:59802, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-12T06:12:31,201 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33455-0x10018b511340002, quorum=127.0.0.1:59802, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-12T06:12:31,202 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33455 2024-12-12T06:12:31,203 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33455 2024-12-12T06:12:31,203 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33455 2024-12-12T06:12:31,204 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33455 2024-12-12T06:12:31,205 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33455 2024-12-12T06:12:31,224 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/ffbfd3107920:0 server-side Connection retries=45 2024-12-12T06:12:31,225 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-12T06:12:31,225 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-12T06:12:31,225 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-12T06:12:31,225 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-12T06:12:31,225 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-12T06:12:31,225 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-12T06:12:31,226 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-12T06:12:31,227 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42275 2024-12-12T06:12:31,229 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42275 connecting to ZooKeeper ensemble=127.0.0.1:59802 2024-12-12T06:12:31,230 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-12T06:12:31,234 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-12T06:12:31,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:422750x0, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-12T06:12:31,258 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:422750x0, quorum=127.0.0.1:59802, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-12T06:12:31,258 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42275-0x10018b511340003 connected 2024-12-12T06:12:31,259 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-12T06:12:31,260 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-12T06:12:31,260 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42275-0x10018b511340003, quorum=127.0.0.1:59802, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-12T06:12:31,262 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42275-0x10018b511340003, quorum=127.0.0.1:59802, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-12T06:12:31,265 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42275 2024-12-12T06:12:31,265 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42275 2024-12-12T06:12:31,266 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42275 2024-12-12T06:12:31,266 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42275 2024-12-12T06:12:31,267 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42275 2024-12-12T06:12:31,283 DEBUG [M:0;ffbfd3107920:45209 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;ffbfd3107920:45209 2024-12-12T06:12:31,284 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/ffbfd3107920,45209,1733983950336 2024-12-12T06:12:31,300 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42275-0x10018b511340003, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-12T06:12:31,300 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33455-0x10018b511340002, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-12T06:12:31,300 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46199-0x10018b511340001, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-12T06:12:31,300 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45209-0x10018b511340000, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-12T06:12:31,304 DEBUG [master/ffbfd3107920:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45209-0x10018b511340000, quorum=127.0.0.1:59802, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/ffbfd3107920,45209,1733983950336 2024-12-12T06:12:31,333 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46199-0x10018b511340001, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-12T06:12:31,333 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42275-0x10018b511340003, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-12T06:12:31,333 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45209-0x10018b511340000, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-12T06:12:31,333 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33455-0x10018b511340002, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-12T06:12:31,333 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46199-0x10018b511340001, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-12T06:12:31,333 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33455-0x10018b511340002, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-12T06:12:31,333 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42275-0x10018b511340003, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-12T06:12:31,334 DEBUG [master/ffbfd3107920:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45209-0x10018b511340000, quorum=127.0.0.1:59802, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-12T06:12:31,335 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/ffbfd3107920,45209,1733983950336 from backup master directory 2024-12-12T06:12:31,344 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46199-0x10018b511340001, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-12T06:12:31,344 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42275-0x10018b511340003, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-12T06:12:31,344 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45209-0x10018b511340000, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/ffbfd3107920,45209,1733983950336 2024-12-12T06:12:31,344 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33455-0x10018b511340002, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-12T06:12:31,344 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45209-0x10018b511340000, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-12T06:12:31,345 WARN [master/ffbfd3107920:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-12T06:12:31,345 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=ffbfd3107920,45209,1733983950336 2024-12-12T06:12:31,349 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-12T06:12:31,350 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-12T06:12:31,406 DEBUG [master/ffbfd3107920:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/hbase.id] with ID: 85203218-ceb3-452a-ac4d-28616485cd2d 2024-12-12T06:12:31,406 DEBUG [master/ffbfd3107920:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/.tmp/hbase.id 2024-12-12T06:12:31,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42285 is added to blk_1073741826_1002 (size=42) 2024-12-12T06:12:31,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38541 is added to blk_1073741826_1002 (size=42) 2024-12-12T06:12:31,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43493 is added to blk_1073741826_1002 (size=42) 2024-12-12T06:12:31,423 DEBUG [master/ffbfd3107920:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/.tmp/hbase.id]:[hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/hbase.id] 2024-12-12T06:12:31,467 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-12T06:12:31,471 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-12T06:12:31,487 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 14ms. 2024-12-12T06:12:31,519 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46199-0x10018b511340001, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-12T06:12:31,519 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42275-0x10018b511340003, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-12T06:12:31,519 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45209-0x10018b511340000, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-12T06:12:31,519 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33455-0x10018b511340002, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-12T06:12:31,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38541 is added to blk_1073741827_1003 (size=196) 2024-12-12T06:12:31,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43493 is added to blk_1073741827_1003 (size=196) 2024-12-12T06:12:31,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42285 is added to blk_1073741827_1003 (size=196) 2024-12-12T06:12:31,555 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-12T06:12:31,557 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-12T06:12:31,570 DEBUG [master/ffbfd3107920:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-12T06:12:31,574 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-12T06:12:31,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42285 is added to blk_1073741828_1004 (size=1189) 2024-12-12T06:12:31,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43493 is added to blk_1073741828_1004 (size=1189) 2024-12-12T06:12:31,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38541 is added to blk_1073741828_1004 (size=1189) 2024-12-12T06:12:31,621 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/MasterData/data/master/store 2024-12-12T06:12:31,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43493 is added to blk_1073741829_1005 (size=34) 2024-12-12T06:12:31,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42285 is added to blk_1073741829_1005 (size=34) 2024-12-12T06:12:31,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38541 is added to blk_1073741829_1005 (size=34) 2024-12-12T06:12:31,647 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-12T06:12:31,650 DEBUG [master/ffbfd3107920:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-12T06:12:31,651 DEBUG [master/ffbfd3107920:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-12T06:12:31,652 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-12T06:12:31,652 DEBUG [master/ffbfd3107920:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-12T06:12:31,654 DEBUG [master/ffbfd3107920:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-12T06:12:31,654 DEBUG [master/ffbfd3107920:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-12T06:12:31,654 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-12T06:12:31,655 DEBUG [master/ffbfd3107920:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733983951651Disabling compacts and flushes for region at 1733983951651Disabling writes for close at 1733983951654 (+3 ms)Writing region close event to WAL at 1733983951654Closed at 1733983951654 2024-12-12T06:12:31,657 WARN [master/ffbfd3107920:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/MasterData/data/master/store/.initializing 2024-12-12T06:12:31,657 DEBUG [master/ffbfd3107920:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/MasterData/WALs/ffbfd3107920,45209,1733983950336 2024-12-12T06:12:31,665 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-12T06:12:31,677 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ffbfd3107920%2C45209%2C1733983950336, suffix=, logDir=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/MasterData/WALs/ffbfd3107920,45209,1733983950336, archiveDir=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/MasterData/oldWALs, maxLogs=10 2024-12-12T06:12:31,698 DEBUG [master/ffbfd3107920:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/MasterData/WALs/ffbfd3107920,45209,1733983950336/ffbfd3107920%2C45209%2C1733983950336.1733983951681, exclude list is [], retry=0 2024-12-12T06:12:31,713 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42285,DS-6cdf379f-c6b3-472d-9e31-f23a652ce941,DISK] 2024-12-12T06:12:31,713 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43493,DS-bf6839c4-19ca-45ad-a024-f2d33b255052,DISK] 2024-12-12T06:12:31,713 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38541,DS-2e86db76-0bd9-4ac5-b9f1-045da95823f2,DISK] 2024-12-12T06:12:31,716 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-12T06:12:31,750 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/MasterData/WALs/ffbfd3107920,45209,1733983950336/ffbfd3107920%2C45209%2C1733983950336.1733983951681 2024-12-12T06:12:31,751 DEBUG [master/ffbfd3107920:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44435:44435),(127.0.0.1/127.0.0.1:39173:39173),(127.0.0.1/127.0.0.1:42655:42655)] 2024-12-12T06:12:31,751 DEBUG [master/ffbfd3107920:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-12T06:12:31,752 DEBUG [master/ffbfd3107920:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-12T06:12:31,755 DEBUG [master/ffbfd3107920:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-12T06:12:31,756 DEBUG [master/ffbfd3107920:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-12T06:12:31,788 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-12T06:12:31,809 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-12T06:12:31,812 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-12T06:12:31,815 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-12T06:12:31,815 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-12T06:12:31,818 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-12T06:12:31,819 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-12T06:12:31,820 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-12T06:12:31,820 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-12T06:12:31,823 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-12T06:12:31,823 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-12T06:12:31,824 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-12T06:12:31,825 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-12T06:12:31,827 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-12T06:12:31,827 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-12T06:12:31,828 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-12T06:12:31,829 DEBUG [master/ffbfd3107920:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-12T06:12:31,832 DEBUG [master/ffbfd3107920:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-12T06:12:31,833 DEBUG [master/ffbfd3107920:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-12T06:12:31,838 DEBUG [master/ffbfd3107920:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-12T06:12:31,839 DEBUG [master/ffbfd3107920:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-12T06:12:31,842 DEBUG [master/ffbfd3107920:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-12T06:12:31,846 DEBUG [master/ffbfd3107920:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-12T06:12:31,851 DEBUG [master/ffbfd3107920:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-12T06:12:31,852 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70344181, jitterRate=0.04820998013019562}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-12T06:12:31,857 DEBUG [master/ffbfd3107920:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733983951767Initializing all the Stores at 1733983951769 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733983951770 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733983951771 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733983951771Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733983951771Cleaning up temporary data from old regions at 1733983951839 (+68 ms)Region opened successfully at 1733983951857 (+18 ms) 2024-12-12T06:12:31,859 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-12T06:12:31,891 DEBUG [master/ffbfd3107920:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d6c4f82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ffbfd3107920/172.17.0.2:0 2024-12-12T06:12:31,922 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-12T06:12:31,931 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-12T06:12:31,931 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-12T06:12:31,934 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-12T06:12:31,935 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-12T06:12:31,940 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-12T06:12:31,941 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-12T06:12:31,962 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-12T06:12:31,970 DEBUG [master/ffbfd3107920:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45209-0x10018b511340000, quorum=127.0.0.1:59802, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-12T06:12:32,010 DEBUG [master/ffbfd3107920:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-12T06:12:32,013 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-12T06:12:32,015 DEBUG [master/ffbfd3107920:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45209-0x10018b511340000, quorum=127.0.0.1:59802, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-12T06:12:32,027 DEBUG [master/ffbfd3107920:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-12T06:12:32,030 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-12T06:12:32,034 DEBUG [master/ffbfd3107920:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45209-0x10018b511340000, quorum=127.0.0.1:59802, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-12T06:12:32,043 DEBUG [master/ffbfd3107920:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-12T06:12:32,045 DEBUG [master/ffbfd3107920:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45209-0x10018b511340000, quorum=127.0.0.1:59802, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-12T06:12:32,052 DEBUG [master/ffbfd3107920:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-12T06:12:32,073 DEBUG [master/ffbfd3107920:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45209-0x10018b511340000, quorum=127.0.0.1:59802, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-12T06:12:32,082 DEBUG [master/ffbfd3107920:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-12T06:12:32,094 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45209-0x10018b511340000, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-12T06:12:32,094 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33455-0x10018b511340002, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-12T06:12:32,094 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46199-0x10018b511340001, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-12T06:12:32,094 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42275-0x10018b511340003, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-12T06:12:32,094 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45209-0x10018b511340000, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-12T06:12:32,094 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46199-0x10018b511340001, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-12T06:12:32,094 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42275-0x10018b511340003, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-12T06:12:32,094 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33455-0x10018b511340002, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-12T06:12:32,098 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=ffbfd3107920,45209,1733983950336, sessionid=0x10018b511340000, setting cluster-up flag (Was=false) 2024-12-12T06:12:32,127 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33455-0x10018b511340002, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-12T06:12:32,127 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46199-0x10018b511340001, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-12T06:12:32,127 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45209-0x10018b511340000, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-12T06:12:32,127 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42275-0x10018b511340003, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-12T06:12:32,152 DEBUG [master/ffbfd3107920:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-12T06:12:32,157 DEBUG [master/ffbfd3107920:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ffbfd3107920,45209,1733983950336 2024-12-12T06:12:32,177 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33455-0x10018b511340002, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-12T06:12:32,177 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45209-0x10018b511340000, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-12T06:12:32,177 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42275-0x10018b511340003, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-12T06:12:32,177 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46199-0x10018b511340001, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-12T06:12:32,202 DEBUG [master/ffbfd3107920:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-12T06:12:32,205 DEBUG [master/ffbfd3107920:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ffbfd3107920,45209,1733983950336 2024-12-12T06:12:32,214 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-12T06:12:32,271 INFO [RS:1;ffbfd3107920:33455 {}] regionserver.HRegionServer(746): ClusterId : 85203218-ceb3-452a-ac4d-28616485cd2d 2024-12-12T06:12:32,271 INFO [RS:0;ffbfd3107920:46199 {}] regionserver.HRegionServer(746): ClusterId : 85203218-ceb3-452a-ac4d-28616485cd2d 2024-12-12T06:12:32,271 INFO [RS:2;ffbfd3107920:42275 {}] regionserver.HRegionServer(746): ClusterId : 85203218-ceb3-452a-ac4d-28616485cd2d 2024-12-12T06:12:32,273 DEBUG [RS:0;ffbfd3107920:46199 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-12T06:12:32,273 DEBUG [RS:2;ffbfd3107920:42275 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-12T06:12:32,273 DEBUG [RS:1;ffbfd3107920:33455 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-12T06:12:32,275 DEBUG [master/ffbfd3107920:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-12T06:12:32,284 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-12T06:12:32,290 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-12T06:12:32,296 DEBUG [RS:1;ffbfd3107920:33455 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-12T06:12:32,296 DEBUG [RS:0;ffbfd3107920:46199 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-12T06:12:32,296 DEBUG [RS:2;ffbfd3107920:42275 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-12T06:12:32,296 DEBUG [RS:1;ffbfd3107920:33455 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-12T06:12:32,296 DEBUG [RS:0;ffbfd3107920:46199 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-12T06:12:32,296 DEBUG [RS:2;ffbfd3107920:42275 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-12T06:12:32,296 DEBUG [master/ffbfd3107920:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: ffbfd3107920,45209,1733983950336 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-12T06:12:32,309 DEBUG [RS:0;ffbfd3107920:46199 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-12T06:12:32,309 DEBUG [RS:1;ffbfd3107920:33455 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-12T06:12:32,309 DEBUG [RS:2;ffbfd3107920:42275 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-12T06:12:32,309 DEBUG [RS:0;ffbfd3107920:46199 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27092c04, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ffbfd3107920/172.17.0.2:0 2024-12-12T06:12:32,309 DEBUG [RS:1;ffbfd3107920:33455 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59c5dd2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ffbfd3107920/172.17.0.2:0 2024-12-12T06:12:32,309 DEBUG [RS:2;ffbfd3107920:42275 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@657851d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ffbfd3107920/172.17.0.2:0 2024-12-12T06:12:32,311 DEBUG [master/ffbfd3107920:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/ffbfd3107920:0, corePoolSize=5, maxPoolSize=5 2024-12-12T06:12:32,311 DEBUG [master/ffbfd3107920:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/ffbfd3107920:0, corePoolSize=5, maxPoolSize=5 2024-12-12T06:12:32,311 DEBUG [master/ffbfd3107920:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/ffbfd3107920:0, corePoolSize=5, maxPoolSize=5 2024-12-12T06:12:32,311 DEBUG [master/ffbfd3107920:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/ffbfd3107920:0, corePoolSize=5, maxPoolSize=5 2024-12-12T06:12:32,311 DEBUG [master/ffbfd3107920:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/ffbfd3107920:0, corePoolSize=10, maxPoolSize=10 2024-12-12T06:12:32,311 DEBUG [master/ffbfd3107920:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/ffbfd3107920:0, corePoolSize=1, maxPoolSize=1 2024-12-12T06:12:32,311 DEBUG [master/ffbfd3107920:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/ffbfd3107920:0, corePoolSize=2, maxPoolSize=2 2024-12-12T06:12:32,312 DEBUG [master/ffbfd3107920:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/ffbfd3107920:0, corePoolSize=1, maxPoolSize=1 2024-12-12T06:12:32,313 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733983982313 2024-12-12T06:12:32,315 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-12T06:12:32,316 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-12T06:12:32,318 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-12T06:12:32,318 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-12T06:12:32,320 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-12T06:12:32,320 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-12T06:12:32,320 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-12T06:12:32,321 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-12T06:12:32,321 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-12T06:12:32,325 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-12T06:12:32,325 DEBUG [RS:0;ffbfd3107920:46199 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;ffbfd3107920:46199 2024-12-12T06:12:32,326 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-12T06:12:32,326 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-12T06:12:32,326 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-12T06:12:32,326 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-12T06:12:32,328 DEBUG [RS:2;ffbfd3107920:42275 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;ffbfd3107920:42275 2024-12-12T06:12:32,328 INFO [RS:0;ffbfd3107920:46199 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-12T06:12:32,328 INFO [RS:2;ffbfd3107920:42275 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-12T06:12:32,329 INFO [RS:0;ffbfd3107920:46199 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-12T06:12:32,329 INFO [RS:2;ffbfd3107920:42275 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-12T06:12:32,329 DEBUG [RS:0;ffbfd3107920:46199 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-12T06:12:32,329 DEBUG [RS:2;ffbfd3107920:42275 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-12T06:12:32,329 DEBUG [RS:1;ffbfd3107920:33455 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;ffbfd3107920:33455 2024-12-12T06:12:32,329 INFO [RS:1;ffbfd3107920:33455 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-12T06:12:32,329 INFO [RS:1;ffbfd3107920:33455 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-12T06:12:32,329 DEBUG [RS:1;ffbfd3107920:33455 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-12T06:12:32,330 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-12T06:12:32,330 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-12T06:12:32,332 INFO [RS:2;ffbfd3107920:42275 {}] regionserver.HRegionServer(2659): reportForDuty to master=ffbfd3107920,45209,1733983950336 with port=42275, startcode=1733983951224 2024-12-12T06:12:32,332 INFO [RS:1;ffbfd3107920:33455 {}] regionserver.HRegionServer(2659): reportForDuty to master=ffbfd3107920,45209,1733983950336 with port=33455, startcode=1733983951151 2024-12-12T06:12:32,332 DEBUG [master/ffbfd3107920:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/ffbfd3107920:0:becomeActiveMaster-HFileCleaner.large.0-1733983952331,5,FailOnTimeoutGroup] 2024-12-12T06:12:32,332 INFO [RS:0;ffbfd3107920:46199 {}] regionserver.HRegionServer(2659): reportForDuty to master=ffbfd3107920,45209,1733983950336 with port=46199, startcode=1733983951048 2024-12-12T06:12:32,335 DEBUG [master/ffbfd3107920:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/ffbfd3107920:0:becomeActiveMaster-HFileCleaner.small.0-1733983952332,5,FailOnTimeoutGroup] 2024-12-12T06:12:32,336 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-12T06:12:32,336 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-12T06:12:32,337 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-12T06:12:32,338 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-12T06:12:32,345 DEBUG [RS:0;ffbfd3107920:46199 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-12T06:12:32,345 DEBUG [RS:1;ffbfd3107920:33455 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-12T06:12:32,345 DEBUG [RS:2;ffbfd3107920:42275 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-12T06:12:32,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43493 is added to blk_1073741831_1007 (size=1321) 2024-12-12T06:12:32,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42285 is added to blk_1073741831_1007 (size=1321) 2024-12-12T06:12:32,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38541 is added to blk_1073741831_1007 (size=1321) 2024-12-12T06:12:32,393 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-12T06:12:32,393 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0 2024-12-12T06:12:32,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42285 is added to blk_1073741832_1008 (size=32) 2024-12-12T06:12:32,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43493 is added to blk_1073741832_1008 (size=32) 2024-12-12T06:12:32,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38541 is added to blk_1073741832_1008 (size=32) 2024-12-12T06:12:32,411 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-12T06:12:32,411 INFO [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45841, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-12-12T06:12:32,411 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45249, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-12-12T06:12:32,411 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35199, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-12T06:12:32,414 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-12T06:12:32,416 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-12T06:12:32,416 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-12T06:12:32,416 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45209 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ffbfd3107920,33455,1733983951151 2024-12-12T06:12:32,417 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-12T06:12:32,418 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-12T06:12:32,419 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45209 {}] master.ServerManager(517): Registering regionserver=ffbfd3107920,33455,1733983951151 2024-12-12T06:12:32,421 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-12T06:12:32,421 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-12T06:12:32,422 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-12T06:12:32,423 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-12T06:12:32,426 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-12T06:12:32,426 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-12T06:12:32,427 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-12T06:12:32,427 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-12T06:12:32,430 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-12T06:12:32,430 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-12T06:12:32,431 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-12T06:12:32,431 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45209 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ffbfd3107920,46199,1733983951048 2024-12-12T06:12:32,431 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-12T06:12:32,431 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45209 {}] master.ServerManager(517): Registering regionserver=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:32,433 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/hbase/meta/1588230740 2024-12-12T06:12:32,434 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/hbase/meta/1588230740 2024-12-12T06:12:32,435 DEBUG [RS:1;ffbfd3107920:33455 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0 2024-12-12T06:12:32,435 DEBUG [RS:1;ffbfd3107920:33455 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38031 2024-12-12T06:12:32,435 DEBUG [RS:1;ffbfd3107920:33455 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-12T06:12:32,436 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45209 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ffbfd3107920,42275,1733983951224 2024-12-12T06:12:32,436 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45209 {}] master.ServerManager(517): Registering regionserver=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:32,436 DEBUG [RS:0;ffbfd3107920:46199 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0 2024-12-12T06:12:32,436 DEBUG [RS:0;ffbfd3107920:46199 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38031 2024-12-12T06:12:32,436 DEBUG [RS:0;ffbfd3107920:46199 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-12T06:12:32,437 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-12T06:12:32,437 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-12T06:12:32,439 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-12T06:12:32,440 DEBUG [RS:2;ffbfd3107920:42275 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0 2024-12-12T06:12:32,440 DEBUG [RS:2;ffbfd3107920:42275 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38031 2024-12-12T06:12:32,440 DEBUG [RS:2;ffbfd3107920:42275 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-12T06:12:32,441 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-12T06:12:32,445 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-12T06:12:32,446 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69589914, jitterRate=0.03697052597999573}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-12T06:12:32,449 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733983952411Initializing all the Stores at 1733983952413 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733983952413Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733983952413Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733983952413Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733983952414 (+1 ms)Cleaning up temporary data from old regions at 1733983952438 (+24 ms)Region opened successfully at 1733983952449 (+11 ms) 2024-12-12T06:12:32,449 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-12T06:12:32,449 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-12T06:12:32,449 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-12T06:12:32,449 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-12T06:12:32,449 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-12T06:12:32,451 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-12T06:12:32,451 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733983952449Disabling compacts and flushes for region at 1733983952449Disabling writes for close at 1733983952449Writing region close event to WAL at 1733983952450 (+1 ms)Closed at 1733983952451 (+1 ms) 2024-12-12T06:12:32,454 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-12T06:12:32,454 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-12T06:12:32,459 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-12T06:12:32,469 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-12T06:12:32,471 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-12T06:12:32,491 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45209-0x10018b511340000, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-12T06:12:32,508 DEBUG [RS:1;ffbfd3107920:33455 {}] zookeeper.ZKUtil(111): regionserver:33455-0x10018b511340002, quorum=127.0.0.1:59802, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ffbfd3107920,33455,1733983951151 2024-12-12T06:12:32,508 DEBUG [RS:0;ffbfd3107920:46199 {}] zookeeper.ZKUtil(111): regionserver:46199-0x10018b511340001, quorum=127.0.0.1:59802, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ffbfd3107920,46199,1733983951048 2024-12-12T06:12:32,508 WARN [RS:1;ffbfd3107920:33455 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-12T06:12:32,508 WARN [RS:0;ffbfd3107920:46199 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-12T06:12:32,508 INFO [RS:1;ffbfd3107920:33455 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-12T06:12:32,508 INFO [RS:0;ffbfd3107920:46199 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-12T06:12:32,509 DEBUG [RS:2;ffbfd3107920:42275 {}] zookeeper.ZKUtil(111): regionserver:42275-0x10018b511340003, quorum=127.0.0.1:59802, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ffbfd3107920,42275,1733983951224 2024-12-12T06:12:32,509 DEBUG [RS:1;ffbfd3107920:33455 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/WALs/ffbfd3107920,33455,1733983951151 2024-12-12T06:12:32,509 DEBUG [RS:0;ffbfd3107920:46199 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/WALs/ffbfd3107920,46199,1733983951048 2024-12-12T06:12:32,509 WARN [RS:2;ffbfd3107920:42275 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-12T06:12:32,509 INFO [RS:2;ffbfd3107920:42275 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-12T06:12:32,509 DEBUG [RS:2;ffbfd3107920:42275 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/WALs/ffbfd3107920,42275,1733983951224 2024-12-12T06:12:32,511 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ffbfd3107920,46199,1733983951048] 2024-12-12T06:12:32,511 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ffbfd3107920,33455,1733983951151] 2024-12-12T06:12:32,511 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ffbfd3107920,42275,1733983951224] 2024-12-12T06:12:32,539 INFO [RS:1;ffbfd3107920:33455 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-12T06:12:32,539 INFO [RS:2;ffbfd3107920:42275 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-12T06:12:32,539 INFO [RS:0;ffbfd3107920:46199 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-12T06:12:32,550 INFO [RS:0;ffbfd3107920:46199 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-12T06:12:32,550 INFO [RS:2;ffbfd3107920:42275 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-12T06:12:32,550 INFO [RS:1;ffbfd3107920:33455 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-12T06:12:32,555 INFO [RS:0;ffbfd3107920:46199 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-12T06:12:32,555 INFO [RS:2;ffbfd3107920:42275 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-12T06:12:32,555 INFO [RS:1;ffbfd3107920:33455 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-12T06:12:32,555 INFO [RS:0;ffbfd3107920:46199 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-12T06:12:32,555 INFO [RS:2;ffbfd3107920:42275 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-12T06:12:32,555 INFO [RS:1;ffbfd3107920:33455 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-12T06:12:32,556 INFO [RS:0;ffbfd3107920:46199 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-12T06:12:32,556 INFO [RS:2;ffbfd3107920:42275 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-12T06:12:32,556 INFO [RS:1;ffbfd3107920:33455 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-12T06:12:32,561 INFO [RS:1;ffbfd3107920:33455 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-12T06:12:32,561 INFO [RS:0;ffbfd3107920:46199 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-12T06:12:32,561 INFO [RS:2;ffbfd3107920:42275 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-12T06:12:32,563 INFO [RS:0;ffbfd3107920:46199 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-12T06:12:32,563 INFO [RS:1;ffbfd3107920:33455 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-12T06:12:32,563 INFO [RS:2;ffbfd3107920:42275 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-12T06:12:32,564 DEBUG [RS:1;ffbfd3107920:33455 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ffbfd3107920:0, corePoolSize=1, maxPoolSize=1 2024-12-12T06:12:32,564 DEBUG [RS:0;ffbfd3107920:46199 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ffbfd3107920:0, corePoolSize=1, maxPoolSize=1 2024-12-12T06:12:32,564 DEBUG [RS:2;ffbfd3107920:42275 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ffbfd3107920:0, corePoolSize=1, maxPoolSize=1 2024-12-12T06:12:32,564 DEBUG [RS:0;ffbfd3107920:46199 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ffbfd3107920:0, corePoolSize=1, maxPoolSize=1 2024-12-12T06:12:32,564 DEBUG [RS:1;ffbfd3107920:33455 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ffbfd3107920:0, corePoolSize=1, maxPoolSize=1 2024-12-12T06:12:32,564 DEBUG [RS:2;ffbfd3107920:42275 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ffbfd3107920:0, corePoolSize=1, maxPoolSize=1 2024-12-12T06:12:32,564 DEBUG [RS:0;ffbfd3107920:46199 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ffbfd3107920:0, corePoolSize=1, maxPoolSize=1 2024-12-12T06:12:32,564 DEBUG [RS:1;ffbfd3107920:33455 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ffbfd3107920:0, corePoolSize=1, maxPoolSize=1 2024-12-12T06:12:32,564 DEBUG [RS:0;ffbfd3107920:46199 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ffbfd3107920:0, corePoolSize=1, maxPoolSize=1 2024-12-12T06:12:32,564 DEBUG [RS:2;ffbfd3107920:42275 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ffbfd3107920:0, corePoolSize=1, maxPoolSize=1 2024-12-12T06:12:32,564 DEBUG [RS:0;ffbfd3107920:46199 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ffbfd3107920:0, corePoolSize=1, maxPoolSize=1 2024-12-12T06:12:32,564 DEBUG [RS:1;ffbfd3107920:33455 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ffbfd3107920:0, corePoolSize=1, maxPoolSize=1 2024-12-12T06:12:32,564 DEBUG [RS:2;ffbfd3107920:42275 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ffbfd3107920:0, corePoolSize=1, maxPoolSize=1 2024-12-12T06:12:32,564 DEBUG [RS:0;ffbfd3107920:46199 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ffbfd3107920:0, corePoolSize=2, maxPoolSize=2 2024-12-12T06:12:32,564 DEBUG [RS:1;ffbfd3107920:33455 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ffbfd3107920:0, corePoolSize=1, maxPoolSize=1 2024-12-12T06:12:32,564 DEBUG [RS:0;ffbfd3107920:46199 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ffbfd3107920:0, corePoolSize=1, maxPoolSize=1 2024-12-12T06:12:32,564 DEBUG [RS:2;ffbfd3107920:42275 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ffbfd3107920:0, corePoolSize=1, maxPoolSize=1 2024-12-12T06:12:32,564 DEBUG [RS:1;ffbfd3107920:33455 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ffbfd3107920:0, corePoolSize=2, maxPoolSize=2 2024-12-12T06:12:32,565 DEBUG [RS:0;ffbfd3107920:46199 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ffbfd3107920:0, corePoolSize=1, maxPoolSize=1 2024-12-12T06:12:32,565 DEBUG [RS:2;ffbfd3107920:42275 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ffbfd3107920:0, corePoolSize=2, maxPoolSize=2 2024-12-12T06:12:32,565 DEBUG [RS:1;ffbfd3107920:33455 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ffbfd3107920:0, corePoolSize=1, maxPoolSize=1 2024-12-12T06:12:32,565 DEBUG [RS:0;ffbfd3107920:46199 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ffbfd3107920:0, corePoolSize=1, maxPoolSize=1 2024-12-12T06:12:32,565 DEBUG [RS:2;ffbfd3107920:42275 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ffbfd3107920:0, corePoolSize=1, maxPoolSize=1 2024-12-12T06:12:32,565 DEBUG [RS:0;ffbfd3107920:46199 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ffbfd3107920:0, corePoolSize=1, maxPoolSize=1 2024-12-12T06:12:32,565 DEBUG [RS:1;ffbfd3107920:33455 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ffbfd3107920:0, corePoolSize=1, maxPoolSize=1 2024-12-12T06:12:32,565 DEBUG [RS:0;ffbfd3107920:46199 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ffbfd3107920:0, corePoolSize=1, maxPoolSize=1 2024-12-12T06:12:32,565 DEBUG [RS:2;ffbfd3107920:42275 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ffbfd3107920:0, corePoolSize=1, maxPoolSize=1 2024-12-12T06:12:32,565 DEBUG [RS:1;ffbfd3107920:33455 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ffbfd3107920:0, corePoolSize=1, maxPoolSize=1 2024-12-12T06:12:32,565 DEBUG [RS:0;ffbfd3107920:46199 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ffbfd3107920:0, corePoolSize=1, maxPoolSize=1 2024-12-12T06:12:32,565 DEBUG [RS:2;ffbfd3107920:42275 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ffbfd3107920:0, corePoolSize=1, maxPoolSize=1 2024-12-12T06:12:32,565 DEBUG [RS:1;ffbfd3107920:33455 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ffbfd3107920:0, corePoolSize=1, maxPoolSize=1 2024-12-12T06:12:32,565 DEBUG [RS:0;ffbfd3107920:46199 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ffbfd3107920:0, corePoolSize=3, maxPoolSize=3 2024-12-12T06:12:32,565 DEBUG [RS:0;ffbfd3107920:46199 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ffbfd3107920:0, corePoolSize=3, maxPoolSize=3 2024-12-12T06:12:32,565 DEBUG [RS:2;ffbfd3107920:42275 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ffbfd3107920:0, corePoolSize=1, maxPoolSize=1 2024-12-12T06:12:32,565 DEBUG [RS:1;ffbfd3107920:33455 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ffbfd3107920:0, corePoolSize=1, maxPoolSize=1 2024-12-12T06:12:32,566 DEBUG [RS:1;ffbfd3107920:33455 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ffbfd3107920:0, corePoolSize=1, maxPoolSize=1 2024-12-12T06:12:32,566 DEBUG [RS:2;ffbfd3107920:42275 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ffbfd3107920:0, corePoolSize=1, maxPoolSize=1 2024-12-12T06:12:32,566 DEBUG [RS:1;ffbfd3107920:33455 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ffbfd3107920:0, corePoolSize=3, maxPoolSize=3 2024-12-12T06:12:32,566 DEBUG [RS:2;ffbfd3107920:42275 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ffbfd3107920:0, corePoolSize=1, maxPoolSize=1 2024-12-12T06:12:32,566 DEBUG [RS:1;ffbfd3107920:33455 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ffbfd3107920:0, corePoolSize=3, maxPoolSize=3 2024-12-12T06:12:32,566 DEBUG [RS:2;ffbfd3107920:42275 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ffbfd3107920:0, corePoolSize=3, maxPoolSize=3 2024-12-12T06:12:32,566 DEBUG [RS:2;ffbfd3107920:42275 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ffbfd3107920:0, corePoolSize=3, maxPoolSize=3 2024-12-12T06:12:32,568 INFO [RS:0;ffbfd3107920:46199 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-12T06:12:32,569 INFO [RS:0;ffbfd3107920:46199 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-12T06:12:32,569 INFO [RS:0;ffbfd3107920:46199 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-12T06:12:32,569 INFO [RS:0;ffbfd3107920:46199 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-12T06:12:32,569 INFO [RS:2;ffbfd3107920:42275 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-12T06:12:32,569 INFO [RS:0;ffbfd3107920:46199 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-12T06:12:32,569 INFO [RS:2;ffbfd3107920:42275 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-12T06:12:32,569 INFO [RS:0;ffbfd3107920:46199 {}] hbase.ChoreService(168): Chore ScheduledChore name=ffbfd3107920,46199,1733983951048-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-12T06:12:32,569 INFO [RS:2;ffbfd3107920:42275 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-12T06:12:32,570 INFO [RS:2;ffbfd3107920:42275 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-12T06:12:32,570 INFO [RS:2;ffbfd3107920:42275 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-12T06:12:32,570 INFO [RS:2;ffbfd3107920:42275 {}] hbase.ChoreService(168): Chore ScheduledChore name=ffbfd3107920,42275,1733983951224-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-12T06:12:32,570 INFO [RS:1;ffbfd3107920:33455 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-12T06:12:32,570 INFO [RS:1;ffbfd3107920:33455 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-12T06:12:32,570 INFO [RS:1;ffbfd3107920:33455 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-12T06:12:32,570 INFO [RS:1;ffbfd3107920:33455 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-12T06:12:32,570 INFO [RS:1;ffbfd3107920:33455 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-12T06:12:32,570 INFO [RS:1;ffbfd3107920:33455 {}] hbase.ChoreService(168): Chore ScheduledChore name=ffbfd3107920,33455,1733983951151-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-12T06:12:32,591 INFO [RS:2;ffbfd3107920:42275 {}] regionserver.HeapMemoryManager(213): Starting, tuneOn=false 2024-12-12T06:12:32,591 INFO [RS:1;ffbfd3107920:33455 {}] regionserver.HeapMemoryManager(213): Starting, tuneOn=false 2024-12-12T06:12:32,593 INFO [RS:2;ffbfd3107920:42275 {}] hbase.ChoreService(168): Chore ScheduledChore name=ffbfd3107920,42275,1733983951224-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-12T06:12:32,593 INFO [RS:1;ffbfd3107920:33455 {}] hbase.ChoreService(168): Chore ScheduledChore name=ffbfd3107920,33455,1733983951151-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-12T06:12:32,593 INFO [RS:2;ffbfd3107920:42275 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-12T06:12:32,593 INFO [RS:1;ffbfd3107920:33455 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-12T06:12:32,593 INFO [RS:2;ffbfd3107920:42275 {}] regionserver.Replication(171): ffbfd3107920,42275,1733983951224 started 2024-12-12T06:12:32,593 INFO [RS:1;ffbfd3107920:33455 {}] regionserver.Replication(171): ffbfd3107920,33455,1733983951151 started 2024-12-12T06:12:32,595 INFO [RS:0;ffbfd3107920:46199 {}] regionserver.HeapMemoryManager(213): Starting, tuneOn=false 2024-12-12T06:12:32,596 INFO [RS:0;ffbfd3107920:46199 {}] hbase.ChoreService(168): Chore ScheduledChore name=ffbfd3107920,46199,1733983951048-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-12T06:12:32,596 INFO [RS:0;ffbfd3107920:46199 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-12T06:12:32,596 INFO [RS:0;ffbfd3107920:46199 {}] regionserver.Replication(171): ffbfd3107920,46199,1733983951048 started 2024-12-12T06:12:32,610 INFO [RS:2;ffbfd3107920:42275 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-12T06:12:32,611 INFO [RS:2;ffbfd3107920:42275 {}] regionserver.HRegionServer(1482): Serving as ffbfd3107920,42275,1733983951224, RpcServer on ffbfd3107920/172.17.0.2:42275, sessionid=0x10018b511340003 2024-12-12T06:12:32,612 DEBUG [RS:2;ffbfd3107920:42275 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-12T06:12:32,612 DEBUG [RS:2;ffbfd3107920:42275 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ffbfd3107920,42275,1733983951224 2024-12-12T06:12:32,612 DEBUG [RS:2;ffbfd3107920:42275 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ffbfd3107920,42275,1733983951224' 2024-12-12T06:12:32,612 DEBUG [RS:2;ffbfd3107920:42275 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-12T06:12:32,613 DEBUG [RS:2;ffbfd3107920:42275 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-12T06:12:32,614 DEBUG [RS:2;ffbfd3107920:42275 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-12T06:12:32,614 DEBUG [RS:2;ffbfd3107920:42275 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-12T06:12:32,614 DEBUG [RS:2;ffbfd3107920:42275 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ffbfd3107920,42275,1733983951224 2024-12-12T06:12:32,614 DEBUG [RS:2;ffbfd3107920:42275 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ffbfd3107920,42275,1733983951224' 2024-12-12T06:12:32,614 DEBUG [RS:2;ffbfd3107920:42275 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-12T06:12:32,614 DEBUG [RS:2;ffbfd3107920:42275 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-12T06:12:32,615 DEBUG [RS:2;ffbfd3107920:42275 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-12T06:12:32,615 INFO [RS:2;ffbfd3107920:42275 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-12T06:12:32,615 INFO [RS:2;ffbfd3107920:42275 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-12T06:12:32,617 INFO [RS:1;ffbfd3107920:33455 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-12T06:12:32,617 INFO [RS:1;ffbfd3107920:33455 {}] regionserver.HRegionServer(1482): Serving as ffbfd3107920,33455,1733983951151, RpcServer on ffbfd3107920/172.17.0.2:33455, sessionid=0x10018b511340002 2024-12-12T06:12:32,617 DEBUG [RS:1;ffbfd3107920:33455 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-12T06:12:32,617 DEBUG [RS:1;ffbfd3107920:33455 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ffbfd3107920,33455,1733983951151 2024-12-12T06:12:32,617 DEBUG [RS:1;ffbfd3107920:33455 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ffbfd3107920,33455,1733983951151' 2024-12-12T06:12:32,618 DEBUG [RS:1;ffbfd3107920:33455 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-12T06:12:32,618 INFO [RS:0;ffbfd3107920:46199 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-12T06:12:32,618 INFO [RS:0;ffbfd3107920:46199 {}] regionserver.HRegionServer(1482): Serving as ffbfd3107920,46199,1733983951048, RpcServer on ffbfd3107920/172.17.0.2:46199, sessionid=0x10018b511340001 2024-12-12T06:12:32,618 DEBUG [RS:0;ffbfd3107920:46199 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-12T06:12:32,618 DEBUG [RS:0;ffbfd3107920:46199 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ffbfd3107920,46199,1733983951048 2024-12-12T06:12:32,618 DEBUG [RS:0;ffbfd3107920:46199 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ffbfd3107920,46199,1733983951048' 2024-12-12T06:12:32,618 DEBUG [RS:0;ffbfd3107920:46199 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-12T06:12:32,618 DEBUG [RS:1;ffbfd3107920:33455 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-12T06:12:32,619 DEBUG [RS:0;ffbfd3107920:46199 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-12T06:12:32,619 DEBUG [RS:1;ffbfd3107920:33455 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-12T06:12:32,619 DEBUG [RS:1;ffbfd3107920:33455 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-12T06:12:32,619 DEBUG [RS:1;ffbfd3107920:33455 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ffbfd3107920,33455,1733983951151 2024-12-12T06:12:32,619 DEBUG [RS:1;ffbfd3107920:33455 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ffbfd3107920,33455,1733983951151' 2024-12-12T06:12:32,619 DEBUG [RS:1;ffbfd3107920:33455 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-12T06:12:32,619 DEBUG [RS:0;ffbfd3107920:46199 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-12T06:12:32,619 DEBUG [RS:0;ffbfd3107920:46199 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-12T06:12:32,619 DEBUG [RS:0;ffbfd3107920:46199 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ffbfd3107920,46199,1733983951048 2024-12-12T06:12:32,620 DEBUG [RS:0;ffbfd3107920:46199 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ffbfd3107920,46199,1733983951048' 2024-12-12T06:12:32,620 DEBUG [RS:0;ffbfd3107920:46199 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-12T06:12:32,620 DEBUG [RS:1;ffbfd3107920:33455 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-12T06:12:32,620 DEBUG [RS:0;ffbfd3107920:46199 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-12T06:12:32,620 DEBUG [RS:1;ffbfd3107920:33455 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-12T06:12:32,620 INFO [RS:1;ffbfd3107920:33455 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-12T06:12:32,620 INFO [RS:1;ffbfd3107920:33455 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-12T06:12:32,621 DEBUG [RS:0;ffbfd3107920:46199 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-12T06:12:32,621 INFO [RS:0;ffbfd3107920:46199 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-12T06:12:32,621 INFO [RS:0;ffbfd3107920:46199 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-12T06:12:32,622 WARN [ffbfd3107920:45209 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-12T06:12:32,721 INFO [RS:2;ffbfd3107920:42275 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-12T06:12:32,722 INFO [RS:1;ffbfd3107920:33455 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-12T06:12:32,722 INFO [RS:0;ffbfd3107920:46199 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-12T06:12:32,725 INFO [RS:2;ffbfd3107920:42275 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ffbfd3107920%2C42275%2C1733983951224, suffix=, logDir=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/WALs/ffbfd3107920,42275,1733983951224, archiveDir=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/oldWALs, maxLogs=32 2024-12-12T06:12:32,725 INFO [RS:0;ffbfd3107920:46199 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ffbfd3107920%2C46199%2C1733983951048, suffix=, logDir=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/WALs/ffbfd3107920,46199,1733983951048, archiveDir=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/oldWALs, maxLogs=32 2024-12-12T06:12:32,725 INFO [RS:1;ffbfd3107920:33455 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ffbfd3107920%2C33455%2C1733983951151, suffix=, logDir=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/WALs/ffbfd3107920,33455,1733983951151, archiveDir=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/oldWALs, maxLogs=32 2024-12-12T06:12:32,740 DEBUG [RS:2;ffbfd3107920:42275 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/WALs/ffbfd3107920,42275,1733983951224/ffbfd3107920%2C42275%2C1733983951224.1733983952728, exclude list is [], retry=0 2024-12-12T06:12:32,744 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43493,DS-bf6839c4-19ca-45ad-a024-f2d33b255052,DISK] 2024-12-12T06:12:32,744 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42285,DS-6cdf379f-c6b3-472d-9e31-f23a652ce941,DISK] 2024-12-12T06:12:32,744 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38541,DS-2e86db76-0bd9-4ac5-b9f1-045da95823f2,DISK] 2024-12-12T06:12:32,747 DEBUG [RS:0;ffbfd3107920:46199 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/WALs/ffbfd3107920,46199,1733983951048/ffbfd3107920%2C46199%2C1733983951048.1733983952728, exclude list is [], retry=0 2024-12-12T06:12:32,749 DEBUG [RS:1;ffbfd3107920:33455 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/WALs/ffbfd3107920,33455,1733983951151/ffbfd3107920%2C33455%2C1733983951151.1733983952728, exclude list is [], retry=0 2024-12-12T06:12:32,750 INFO [RS:2;ffbfd3107920:42275 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/WALs/ffbfd3107920,42275,1733983951224/ffbfd3107920%2C42275%2C1733983951224.1733983952728 2024-12-12T06:12:32,750 DEBUG [RS:2;ffbfd3107920:42275 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44435:44435),(127.0.0.1/127.0.0.1:42655:42655),(127.0.0.1/127.0.0.1:39173:39173)] 2024-12-12T06:12:32,754 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42285,DS-6cdf379f-c6b3-472d-9e31-f23a652ce941,DISK] 2024-12-12T06:12:32,754 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42285,DS-6cdf379f-c6b3-472d-9e31-f23a652ce941,DISK] 2024-12-12T06:12:32,754 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43493,DS-bf6839c4-19ca-45ad-a024-f2d33b255052,DISK] 2024-12-12T06:12:32,754 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38541,DS-2e86db76-0bd9-4ac5-b9f1-045da95823f2,DISK] 2024-12-12T06:12:32,755 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43493,DS-bf6839c4-19ca-45ad-a024-f2d33b255052,DISK] 2024-12-12T06:12:32,756 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38541,DS-2e86db76-0bd9-4ac5-b9f1-045da95823f2,DISK] 2024-12-12T06:12:32,761 INFO [RS:1;ffbfd3107920:33455 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/WALs/ffbfd3107920,33455,1733983951151/ffbfd3107920%2C33455%2C1733983951151.1733983952728 2024-12-12T06:12:32,761 INFO [RS:0;ffbfd3107920:46199 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/WALs/ffbfd3107920,46199,1733983951048/ffbfd3107920%2C46199%2C1733983951048.1733983952728 2024-12-12T06:12:32,761 DEBUG [RS:1;ffbfd3107920:33455 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44435:44435),(127.0.0.1/127.0.0.1:42655:42655),(127.0.0.1/127.0.0.1:39173:39173)] 2024-12-12T06:12:32,762 DEBUG [RS:0;ffbfd3107920:46199 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44435:44435),(127.0.0.1/127.0.0.1:39173:39173),(127.0.0.1/127.0.0.1:42655:42655)] 2024-12-12T06:12:32,878 DEBUG [ffbfd3107920:45209 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-12-12T06:12:32,886 DEBUG [ffbfd3107920:45209 {}] balancer.BalancerClusterState(204): Hosts are {ffbfd3107920=0} racks are {/default-rack=0} 2024-12-12T06:12:32,891 DEBUG [ffbfd3107920:45209 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-12T06:12:32,892 DEBUG [ffbfd3107920:45209 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-12T06:12:32,892 DEBUG [ffbfd3107920:45209 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-12T06:12:32,892 DEBUG [ffbfd3107920:45209 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-12T06:12:32,892 DEBUG [ffbfd3107920:45209 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-12T06:12:32,892 DEBUG [ffbfd3107920:45209 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-12T06:12:32,892 INFO [ffbfd3107920:45209 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-12T06:12:32,892 INFO [ffbfd3107920:45209 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-12T06:12:32,892 INFO [ffbfd3107920:45209 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-12T06:12:32,892 DEBUG [ffbfd3107920:45209 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-12T06:12:32,898 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=ffbfd3107920,33455,1733983951151 2024-12-12T06:12:32,904 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ffbfd3107920,33455,1733983951151, state=OPENING 2024-12-12T06:12:32,957 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-12T06:12:32,969 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33455-0x10018b511340002, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-12T06:12:32,969 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46199-0x10018b511340001, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-12T06:12:32,969 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45209-0x10018b511340000, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-12T06:12:32,969 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42275-0x10018b511340003, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-12T06:12:32,972 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:32,972 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:32,973 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:32,973 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:32,975 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-12T06:12:32,978 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=ffbfd3107920,33455,1733983951151}] 2024-12-12T06:12:33,154 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-12T06:12:33,157 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57291, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-12T06:12:33,169 INFO [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-12T06:12:33,169 INFO [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-12T06:12:33,170 INFO [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-12T06:12:33,173 INFO [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ffbfd3107920%2C33455%2C1733983951151.meta, suffix=.meta, logDir=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/WALs/ffbfd3107920,33455,1733983951151, archiveDir=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/oldWALs, maxLogs=32 2024-12-12T06:12:33,189 DEBUG [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/WALs/ffbfd3107920,33455,1733983951151/ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, exclude list is [], retry=0 2024-12-12T06:12:33,194 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43493,DS-bf6839c4-19ca-45ad-a024-f2d33b255052,DISK] 2024-12-12T06:12:33,194 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42285,DS-6cdf379f-c6b3-472d-9e31-f23a652ce941,DISK] 2024-12-12T06:12:33,194 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38541,DS-2e86db76-0bd9-4ac5-b9f1-045da95823f2,DISK] 2024-12-12T06:12:33,196 INFO [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/WALs/ffbfd3107920,33455,1733983951151/ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta 2024-12-12T06:12:33,197 DEBUG [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44435:44435),(127.0.0.1/127.0.0.1:42655:42655),(127.0.0.1/127.0.0.1:39173:39173)] 2024-12-12T06:12:33,197 DEBUG [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-12T06:12:33,198 DEBUG [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-12T06:12:33,200 DEBUG [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-12T06:12:33,205 INFO [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-12T06:12:33,208 DEBUG [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-12T06:12:33,209 DEBUG [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-12T06:12:33,209 DEBUG [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-12T06:12:33,209 DEBUG [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-12T06:12:33,212 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-12T06:12:33,213 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-12T06:12:33,213 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-12T06:12:33,214 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-12T06:12:33,214 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-12T06:12:33,216 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-12T06:12:33,216 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-12T06:12:33,216 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-12T06:12:33,217 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-12T06:12:33,218 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-12T06:12:33,218 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-12T06:12:33,219 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-12T06:12:33,219 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-12T06:12:33,220 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-12T06:12:33,220 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-12T06:12:33,221 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-12T06:12:33,221 DEBUG [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-12T06:12:33,222 DEBUG [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/hbase/meta/1588230740 2024-12-12T06:12:33,224 DEBUG [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/hbase/meta/1588230740 2024-12-12T06:12:33,227 DEBUG [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-12T06:12:33,227 DEBUG [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-12T06:12:33,228 DEBUG [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-12T06:12:33,230 DEBUG [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-12T06:12:33,232 INFO [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73695857, jitterRate=0.09815384447574615}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-12T06:12:33,232 DEBUG [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-12T06:12:33,233 DEBUG [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733983953209Writing region info on filesystem at 1733983953209Initializing all the Stores at 1733983953211 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733983953211Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733983953212 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733983953212Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733983953212Cleaning up temporary data from old regions at 1733983953227 (+15 ms)Running coprocessor post-open hooks at 1733983953232 (+5 ms)Region opened successfully at 1733983953233 (+1 ms) 2024-12-12T06:12:33,239 INFO [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733983953146 2024-12-12T06:12:33,248 DEBUG [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-12T06:12:33,249 INFO [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-12T06:12:33,251 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=ffbfd3107920,33455,1733983951151 2024-12-12T06:12:33,253 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ffbfd3107920,33455,1733983951151, state=OPEN 2024-12-12T06:12:33,290 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33455-0x10018b511340002, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:33,290 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46199-0x10018b511340001, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:33,290 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42275-0x10018b511340003, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:33,291 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45209-0x10018b511340000, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:33,291 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:33,291 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:33,291 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:33,291 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:33,291 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=ffbfd3107920,33455,1733983951151 2024-12-12T06:12:33,298 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-12T06:12:33,298 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ffbfd3107920,33455,1733983951151 in 315 msec 2024-12-12T06:12:33,307 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-12T06:12:33,307 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 841 msec 2024-12-12T06:12:33,309 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-12T06:12:33,309 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-12T06:12:33,325 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-12T06:12:33,326 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ffbfd3107920,33455,1733983951151, seqNum=-1] 2024-12-12T06:12:33,342 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-12T06:12:33,344 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59873, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-12T06:12:33,364 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1230 sec 2024-12-12T06:12:33,364 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733983953364, completionTime=-1 2024-12-12T06:12:33,367 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-12-12T06:12:33,367 DEBUG [master/ffbfd3107920:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-12T06:12:33,417 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-12-12T06:12:33,417 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733984013417 2024-12-12T06:12:33,417 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733984073417 2024-12-12T06:12:33,417 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 50 msec 2024-12-12T06:12:33,419 DEBUG [master/ffbfd3107920:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-12T06:12:33,432 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ffbfd3107920,45209,1733983950336-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-12T06:12:33,432 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ffbfd3107920,45209,1733983950336-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-12T06:12:33,432 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ffbfd3107920,45209,1733983950336-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-12T06:12:33,435 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-ffbfd3107920:45209, period=300000, unit=MILLISECONDS is enabled. 2024-12-12T06:12:33,435 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-12T06:12:33,436 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-12T06:12:33,444 DEBUG [master/ffbfd3107920:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-12T06:12:33,468 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.122sec 2024-12-12T06:12:33,470 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-12T06:12:33,472 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-12T06:12:33,473 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-12T06:12:33,473 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-12T06:12:33,474 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-12T06:12:33,475 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ffbfd3107920,45209,1733983950336-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-12T06:12:33,475 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ffbfd3107920,45209,1733983950336-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-12T06:12:33,481 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6028722b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-12T06:12:33,482 DEBUG [master/ffbfd3107920:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-12T06:12:33,484 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-12T06:12:33,484 INFO [master/ffbfd3107920:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ffbfd3107920,45209,1733983950336-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-12T06:12:33,485 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-12T06:12:33,485 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-12T06:12:33,488 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ffbfd3107920,45209,-1 for getting cluster id 2024-12-12T06:12:33,491 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-12T06:12:33,500 DEBUG [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '85203218-ceb3-452a-ac4d-28616485cd2d' 2024-12-12T06:12:33,503 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-12T06:12:33,503 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "85203218-ceb3-452a-ac4d-28616485cd2d" 2024-12-12T06:12:33,503 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1477bdd9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-12T06:12:33,503 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ffbfd3107920,45209,-1] 2024-12-12T06:12:33,506 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-12T06:12:33,508 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-12T06:12:33,509 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43582, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-12T06:12:33,511 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@110f64a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-12T06:12:33,512 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-12T06:12:33,520 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ffbfd3107920,33455,1733983951151, seqNum=-1] 2024-12-12T06:12:33,520 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-12T06:12:33,523 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50812, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-12T06:12:33,543 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=ffbfd3107920,45209,1733983950336 2024-12-12T06:12:33,547 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-12T06:12:33,552 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is ffbfd3107920,45209,1733983950336 2024-12-12T06:12:33,554 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@247eb1d5 2024-12-12T06:12:33,555 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-12T06:12:33,558 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43594, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-12T06:12:33,565 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45209 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'test', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-12T06:12:33,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45209 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test 2024-12-12T06:12:33,577 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_PRE_OPERATION 2024-12-12T06:12:33,579 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45209 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "test" procId is: 4 2024-12-12T06:12:33,579 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-12T06:12:33,582 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-12T06:12:33,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45209 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-12T06:12:33,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42285 is added to blk_1073741837_1013 (size=358) 2024-12-12T06:12:33,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38541 is added to blk_1073741837_1013 (size=358) 2024-12-12T06:12:33,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43493 is added to blk_1073741837_1013 (size=358) 2024-12-12T06:12:33,607 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8545c1fe97252abe3447dcfaebd5cfa5, NAME => 'test,,1733983953560.8545c1fe97252abe3447dcfaebd5cfa5.', STARTKEY => '', ENDKEY => 'bbb'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0 2024-12-12T06:12:33,608 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 16838c5df0ab4e3938ad84e9a080def9, NAME => 'test,bbb,1733983953560.16838c5df0ab4e3938ad84e9a080def9.', STARTKEY => 'bbb', ENDKEY => 'ccc'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0 2024-12-12T06:12:33,609 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 2eeaafb9a1964f4c6899fa5dde05a463, NAME => 'test,ccc,1733983953560.2eeaafb9a1964f4c6899fa5dde05a463.', STARTKEY => 'ccc', ENDKEY => 'ddd'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0 2024-12-12T06:12:33,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38541 is added to blk_1073741840_1016 (size=45) 2024-12-12T06:12:33,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42285 is added to blk_1073741840_1016 (size=45) 2024-12-12T06:12:33,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43493 is added to blk_1073741840_1016 (size=45) 2024-12-12T06:12:33,633 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ccc,1733983953560.2eeaafb9a1964f4c6899fa5dde05a463.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-12T06:12:33,633 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 2eeaafb9a1964f4c6899fa5dde05a463, disabling compactions & flushes 2024-12-12T06:12:33,633 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ccc,1733983953560.2eeaafb9a1964f4c6899fa5dde05a463. 2024-12-12T06:12:33,633 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1733983953560.2eeaafb9a1964f4c6899fa5dde05a463. 2024-12-12T06:12:33,634 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1733983953560.2eeaafb9a1964f4c6899fa5dde05a463. after waiting 0 ms 2024-12-12T06:12:33,634 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1733983953560.2eeaafb9a1964f4c6899fa5dde05a463. 2024-12-12T06:12:33,634 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ccc,1733983953560.2eeaafb9a1964f4c6899fa5dde05a463. 2024-12-12T06:12:33,634 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 2eeaafb9a1964f4c6899fa5dde05a463: Waiting for close lock at 1733983953633Disabling compacts and flushes for region at 1733983953633Disabling writes for close at 1733983953634 (+1 ms)Writing region close event to WAL at 1733983953634Closed at 1733983953634 2024-12-12T06:12:33,634 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => c98bb859cc9e047b211ddb4c5b967ce5, NAME => 'test,ddd,1733983953560.c98bb859cc9e047b211ddb4c5b967ce5.', STARTKEY => 'ddd', ENDKEY => 'eee'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0 2024-12-12T06:12:33,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43493 is added to blk_1073741838_1014 (size=42) 2024-12-12T06:12:33,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38541 is added to blk_1073741839_1015 (size=45) 2024-12-12T06:12:33,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42285 is added to blk_1073741839_1015 (size=45) 2024-12-12T06:12:33,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38541 is added to blk_1073741838_1014 (size=42) 2024-12-12T06:12:33,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42285 is added to blk_1073741838_1014 (size=42) 2024-12-12T06:12:33,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43493 is added to blk_1073741839_1015 (size=45) 2024-12-12T06:12:33,649 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,,1733983953560.8545c1fe97252abe3447dcfaebd5cfa5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-12T06:12:33,649 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 8545c1fe97252abe3447dcfaebd5cfa5, disabling compactions & flushes 2024-12-12T06:12:33,649 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,,1733983953560.8545c1fe97252abe3447dcfaebd5cfa5. 2024-12-12T06:12:33,649 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1733983953560.8545c1fe97252abe3447dcfaebd5cfa5. 2024-12-12T06:12:33,649 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,,1733983953560.8545c1fe97252abe3447dcfaebd5cfa5. after waiting 0 ms 2024-12-12T06:12:33,649 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,,1733983953560.8545c1fe97252abe3447dcfaebd5cfa5. 2024-12-12T06:12:33,649 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,,1733983953560.8545c1fe97252abe3447dcfaebd5cfa5. 2024-12-12T06:12:33,650 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,bbb,1733983953560.16838c5df0ab4e3938ad84e9a080def9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-12T06:12:33,650 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8545c1fe97252abe3447dcfaebd5cfa5: Waiting for close lock at 1733983953649Disabling compacts and flushes for region at 1733983953649Disabling writes for close at 1733983953649Writing region close event to WAL at 1733983953649Closed at 1733983953649 2024-12-12T06:12:33,650 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 16838c5df0ab4e3938ad84e9a080def9, disabling compactions & flushes 2024-12-12T06:12:33,650 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,bbb,1733983953560.16838c5df0ab4e3938ad84e9a080def9. 2024-12-12T06:12:33,650 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1733983953560.16838c5df0ab4e3938ad84e9a080def9. 2024-12-12T06:12:33,650 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b935ee7eeea02b161c0ebc0d05527e91, NAME => 'test,eee,1733983953560.b935ee7eeea02b161c0ebc0d05527e91.', STARTKEY => 'eee', ENDKEY => 'fff'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0 2024-12-12T06:12:33,651 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1733983953560.16838c5df0ab4e3938ad84e9a080def9. after waiting 0 ms 2024-12-12T06:12:33,651 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1733983953560.16838c5df0ab4e3938ad84e9a080def9. 2024-12-12T06:12:33,651 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,bbb,1733983953560.16838c5df0ab4e3938ad84e9a080def9. 2024-12-12T06:12:33,651 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 16838c5df0ab4e3938ad84e9a080def9: Waiting for close lock at 1733983953650Disabling compacts and flushes for region at 1733983953650Disabling writes for close at 1733983953651 (+1 ms)Writing region close event to WAL at 1733983953651Closed at 1733983953651 2024-12-12T06:12:33,652 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 4584d26a35c36968de829b34ac08fb5d, NAME => 'test,fff,1733983953560.4584d26a35c36968de829b34ac08fb5d.', STARTKEY => 'fff', ENDKEY => 'ggg'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0 2024-12-12T06:12:33,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38541 is added to blk_1073741841_1017 (size=45) 2024-12-12T06:12:33,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42285 is added to blk_1073741841_1017 (size=45) 2024-12-12T06:12:33,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43493 is added to blk_1073741841_1017 (size=45) 2024-12-12T06:12:33,664 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ddd,1733983953560.c98bb859cc9e047b211ddb4c5b967ce5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-12T06:12:33,664 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing c98bb859cc9e047b211ddb4c5b967ce5, disabling compactions & flushes 2024-12-12T06:12:33,664 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ddd,1733983953560.c98bb859cc9e047b211ddb4c5b967ce5. 2024-12-12T06:12:33,664 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1733983953560.c98bb859cc9e047b211ddb4c5b967ce5. 2024-12-12T06:12:33,664 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1733983953560.c98bb859cc9e047b211ddb4c5b967ce5. after waiting 0 ms 2024-12-12T06:12:33,664 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1733983953560.c98bb859cc9e047b211ddb4c5b967ce5. 2024-12-12T06:12:33,665 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ddd,1733983953560.c98bb859cc9e047b211ddb4c5b967ce5. 2024-12-12T06:12:33,665 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for c98bb859cc9e047b211ddb4c5b967ce5: Waiting for close lock at 1733983953664Disabling compacts and flushes for region at 1733983953664Disabling writes for close at 1733983953664Writing region close event to WAL at 1733983953664Closed at 1733983953664 2024-12-12T06:12:33,665 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => b717f65da42dacb67935407f094e8467, NAME => 'test,ggg,1733983953560.b717f65da42dacb67935407f094e8467.', STARTKEY => 'ggg', ENDKEY => 'hhh'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0 2024-12-12T06:12:33,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42285 is added to blk_1073741843_1019 (size=45) 2024-12-12T06:12:33,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38541 is added to blk_1073741843_1019 (size=45) 2024-12-12T06:12:33,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43493 is added to blk_1073741843_1019 (size=45) 2024-12-12T06:12:33,680 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,fff,1733983953560.4584d26a35c36968de829b34ac08fb5d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-12T06:12:33,680 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 4584d26a35c36968de829b34ac08fb5d, disabling compactions & flushes 2024-12-12T06:12:33,680 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,fff,1733983953560.4584d26a35c36968de829b34ac08fb5d. 2024-12-12T06:12:33,680 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1733983953560.4584d26a35c36968de829b34ac08fb5d. 2024-12-12T06:12:33,680 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,fff,1733983953560.4584d26a35c36968de829b34ac08fb5d. after waiting 0 ms 2024-12-12T06:12:33,680 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,fff,1733983953560.4584d26a35c36968de829b34ac08fb5d. 2024-12-12T06:12:33,680 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,fff,1733983953560.4584d26a35c36968de829b34ac08fb5d. 2024-12-12T06:12:33,680 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 4584d26a35c36968de829b34ac08fb5d: Waiting for close lock at 1733983953680Disabling compacts and flushes for region at 1733983953680Disabling writes for close at 1733983953680Writing region close event to WAL at 1733983953680Closed at 1733983953680 2024-12-12T06:12:33,681 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => f2b6a6b11f14670aca9dccceb20d9630, NAME => 'test,hhh,1733983953560.f2b6a6b11f14670aca9dccceb20d9630.', STARTKEY => 'hhh', ENDKEY => 'iii'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0 2024-12-12T06:12:33,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42285 is added to blk_1073741842_1018 (size=45) 2024-12-12T06:12:33,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43493 is added to blk_1073741842_1018 (size=45) 2024-12-12T06:12:33,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38541 is added to blk_1073741842_1018 (size=45) 2024-12-12T06:12:33,685 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,eee,1733983953560.b935ee7eeea02b161c0ebc0d05527e91.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-12T06:12:33,685 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing b935ee7eeea02b161c0ebc0d05527e91, disabling compactions & flushes 2024-12-12T06:12:33,685 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,eee,1733983953560.b935ee7eeea02b161c0ebc0d05527e91. 2024-12-12T06:12:33,685 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1733983953560.b935ee7eeea02b161c0ebc0d05527e91. 2024-12-12T06:12:33,686 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,eee,1733983953560.b935ee7eeea02b161c0ebc0d05527e91. after waiting 0 ms 2024-12-12T06:12:33,686 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,eee,1733983953560.b935ee7eeea02b161c0ebc0d05527e91. 2024-12-12T06:12:33,686 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,eee,1733983953560.b935ee7eeea02b161c0ebc0d05527e91. 2024-12-12T06:12:33,686 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for b935ee7eeea02b161c0ebc0d05527e91: Waiting for close lock at 1733983953685Disabling compacts and flushes for region at 1733983953685Disabling writes for close at 1733983953686 (+1 ms)Writing region close event to WAL at 1733983953686Closed at 1733983953686 2024-12-12T06:12:33,686 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 57995ad3e527ada56b846564833341d1, NAME => 'test,iii,1733983953560.57995ad3e527ada56b846564833341d1.', STARTKEY => 'iii', ENDKEY => 'jjj'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0 2024-12-12T06:12:33,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43493 is added to blk_1073741844_1020 (size=45) 2024-12-12T06:12:33,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38541 is added to blk_1073741844_1020 (size=45) 2024-12-12T06:12:33,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45209 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-12T06:12:33,698 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ggg,1733983953560.b717f65da42dacb67935407f094e8467.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-12T06:12:33,698 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing b717f65da42dacb67935407f094e8467, disabling compactions & flushes 2024-12-12T06:12:33,698 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ggg,1733983953560.b717f65da42dacb67935407f094e8467. 2024-12-12T06:12:33,698 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1733983953560.b717f65da42dacb67935407f094e8467. 2024-12-12T06:12:33,698 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1733983953560.b717f65da42dacb67935407f094e8467. after waiting 0 ms 2024-12-12T06:12:33,699 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1733983953560.b717f65da42dacb67935407f094e8467. 2024-12-12T06:12:33,699 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ggg,1733983953560.b717f65da42dacb67935407f094e8467. 2024-12-12T06:12:33,699 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for b717f65da42dacb67935407f094e8467: Waiting for close lock at 1733983953698Disabling compacts and flushes for region at 1733983953698Disabling writes for close at 1733983953698Writing region close event to WAL at 1733983953699 (+1 ms)Closed at 1733983953699 2024-12-12T06:12:33,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42285 is added to blk_1073741844_1020 (size=45) 2024-12-12T06:12:33,700 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => c77db308a7c1c57fe9ef945e17e28127, NAME => 'test,jjj,1733983953560.c77db308a7c1c57fe9ef945e17e28127.', STARTKEY => 'jjj', ENDKEY => 'kkk'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0 2024-12-12T06:12:33,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42285 is added to blk_1073741846_1022 (size=45) 2024-12-12T06:12:33,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43493 is added to blk_1073741846_1022 (size=45) 2024-12-12T06:12:33,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38541 is added to blk_1073741846_1022 (size=45) 2024-12-12T06:12:33,710 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,iii,1733983953560.57995ad3e527ada56b846564833341d1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-12T06:12:33,711 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 57995ad3e527ada56b846564833341d1, disabling compactions & flushes 2024-12-12T06:12:33,711 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,iii,1733983953560.57995ad3e527ada56b846564833341d1. 2024-12-12T06:12:33,711 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1733983953560.57995ad3e527ada56b846564833341d1. 2024-12-12T06:12:33,711 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,iii,1733983953560.57995ad3e527ada56b846564833341d1. after waiting 0 ms 2024-12-12T06:12:33,711 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,iii,1733983953560.57995ad3e527ada56b846564833341d1. 2024-12-12T06:12:33,711 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,iii,1733983953560.57995ad3e527ada56b846564833341d1. 2024-12-12T06:12:33,711 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 57995ad3e527ada56b846564833341d1: Waiting for close lock at 1733983953711Disabling compacts and flushes for region at 1733983953711Disabling writes for close at 1733983953711Writing region close event to WAL at 1733983953711Closed at 1733983953711 2024-12-12T06:12:33,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43493 is added to blk_1073741845_1021 (size=45) 2024-12-12T06:12:33,712 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 782c52424de7d71cb43b8fbfe8e89f7f, NAME => 'test,kkk,1733983953560.782c52424de7d71cb43b8fbfe8e89f7f.', STARTKEY => 'kkk', ENDKEY => 'lll'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0 2024-12-12T06:12:33,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42285 is added to blk_1073741845_1021 (size=45) 2024-12-12T06:12:33,713 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,hhh,1733983953560.f2b6a6b11f14670aca9dccceb20d9630.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-12T06:12:33,713 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing f2b6a6b11f14670aca9dccceb20d9630, disabling compactions & flushes 2024-12-12T06:12:33,714 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,hhh,1733983953560.f2b6a6b11f14670aca9dccceb20d9630. 2024-12-12T06:12:33,714 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1733983953560.f2b6a6b11f14670aca9dccceb20d9630. 2024-12-12T06:12:33,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38541 is added to blk_1073741845_1021 (size=45) 2024-12-12T06:12:33,714 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1733983953560.f2b6a6b11f14670aca9dccceb20d9630. after waiting 0 ms 2024-12-12T06:12:33,714 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1733983953560.f2b6a6b11f14670aca9dccceb20d9630. 2024-12-12T06:12:33,714 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,hhh,1733983953560.f2b6a6b11f14670aca9dccceb20d9630. 2024-12-12T06:12:33,714 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for f2b6a6b11f14670aca9dccceb20d9630: Waiting for close lock at 1733983953713Disabling compacts and flushes for region at 1733983953713Disabling writes for close at 1733983953714 (+1 ms)Writing region close event to WAL at 1733983953714Closed at 1733983953714 2024-12-12T06:12:33,715 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 5d34bdb8a1414fb25a475fb97fc440b4, NAME => 'test,lll,1733983953560.5d34bdb8a1414fb25a475fb97fc440b4.', STARTKEY => 'lll', ENDKEY => 'mmm'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0 2024-12-12T06:12:33,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43493 is added to blk_1073741847_1023 (size=45) 2024-12-12T06:12:33,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38541 is added to blk_1073741847_1023 (size=45) 2024-12-12T06:12:33,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42285 is added to blk_1073741847_1023 (size=45) 2024-12-12T06:12:33,722 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,jjj,1733983953560.c77db308a7c1c57fe9ef945e17e28127.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-12T06:12:33,722 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing c77db308a7c1c57fe9ef945e17e28127, disabling compactions & flushes 2024-12-12T06:12:33,722 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,jjj,1733983953560.c77db308a7c1c57fe9ef945e17e28127. 2024-12-12T06:12:33,722 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1733983953560.c77db308a7c1c57fe9ef945e17e28127. 2024-12-12T06:12:33,722 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1733983953560.c77db308a7c1c57fe9ef945e17e28127. after waiting 0 ms 2024-12-12T06:12:33,722 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1733983953560.c77db308a7c1c57fe9ef945e17e28127. 2024-12-12T06:12:33,722 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,jjj,1733983953560.c77db308a7c1c57fe9ef945e17e28127. 2024-12-12T06:12:33,722 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for c77db308a7c1c57fe9ef945e17e28127: Waiting for close lock at 1733983953722Disabling compacts and flushes for region at 1733983953722Disabling writes for close at 1733983953722Writing region close event to WAL at 1733983953722Closed at 1733983953722 2024-12-12T06:12:33,723 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => d86220c39cb75f305e7ed4d4c9f1191c, NAME => 'test,mmm,1733983953560.d86220c39cb75f305e7ed4d4c9f1191c.', STARTKEY => 'mmm', ENDKEY => 'nnn'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0 2024-12-12T06:12:33,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38541 is added to blk_1073741848_1024 (size=45) 2024-12-12T06:12:33,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42285 is added to blk_1073741849_1025 (size=45) 2024-12-12T06:12:33,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43493 is added to blk_1073741848_1024 (size=45) 2024-12-12T06:12:33,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38541 is added to blk_1073741849_1025 (size=45) 2024-12-12T06:12:33,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42285 is added to blk_1073741848_1024 (size=45) 2024-12-12T06:12:33,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43493 is added to blk_1073741849_1025 (size=45) 2024-12-12T06:12:33,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42285 is added to blk_1073741850_1026 (size=45) 2024-12-12T06:12:33,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43493 is added to blk_1073741850_1026 (size=45) 2024-12-12T06:12:33,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38541 is added to blk_1073741850_1026 (size=45) 2024-12-12T06:12:33,754 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,mmm,1733983953560.d86220c39cb75f305e7ed4d4c9f1191c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-12T06:12:33,754 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing d86220c39cb75f305e7ed4d4c9f1191c, disabling compactions & flushes 2024-12-12T06:12:33,755 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,mmm,1733983953560.d86220c39cb75f305e7ed4d4c9f1191c. 2024-12-12T06:12:33,755 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1733983953560.d86220c39cb75f305e7ed4d4c9f1191c. 2024-12-12T06:12:33,755 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1733983953560.d86220c39cb75f305e7ed4d4c9f1191c. after waiting 0 ms 2024-12-12T06:12:33,755 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1733983953560.d86220c39cb75f305e7ed4d4c9f1191c. 2024-12-12T06:12:33,755 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,mmm,1733983953560.d86220c39cb75f305e7ed4d4c9f1191c. 2024-12-12T06:12:33,755 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for d86220c39cb75f305e7ed4d4c9f1191c: Waiting for close lock at 1733983953754Disabling compacts and flushes for region at 1733983953754Disabling writes for close at 1733983953755 (+1 ms)Writing region close event to WAL at 1733983953755Closed at 1733983953755 2024-12-12T06:12:33,756 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => be4cb5da32eefc543e09e80b426b6938, NAME => 'test,nnn,1733983953560.be4cb5da32eefc543e09e80b426b6938.', STARTKEY => 'nnn', ENDKEY => 'ooo'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0 2024-12-12T06:12:33,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42285 is added to blk_1073741851_1027 (size=45) 2024-12-12T06:12:33,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38541 is added to blk_1073741851_1027 (size=45) 2024-12-12T06:12:33,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43493 is added to blk_1073741851_1027 (size=45) 2024-12-12T06:12:33,768 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,nnn,1733983953560.be4cb5da32eefc543e09e80b426b6938.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-12T06:12:33,768 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing be4cb5da32eefc543e09e80b426b6938, disabling compactions & flushes 2024-12-12T06:12:33,768 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,nnn,1733983953560.be4cb5da32eefc543e09e80b426b6938. 2024-12-12T06:12:33,768 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1733983953560.be4cb5da32eefc543e09e80b426b6938. 2024-12-12T06:12:33,769 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1733983953560.be4cb5da32eefc543e09e80b426b6938. after waiting 0 ms 2024-12-12T06:12:33,769 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1733983953560.be4cb5da32eefc543e09e80b426b6938. 2024-12-12T06:12:33,769 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,nnn,1733983953560.be4cb5da32eefc543e09e80b426b6938. 2024-12-12T06:12:33,769 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for be4cb5da32eefc543e09e80b426b6938: Waiting for close lock at 1733983953768Disabling compacts and flushes for region at 1733983953768Disabling writes for close at 1733983953769 (+1 ms)Writing region close event to WAL at 1733983953769Closed at 1733983953769 2024-12-12T06:12:33,769 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => a4bf848f8297d89de7b203fcc6678eb5, NAME => 'test,ooo,1733983953560.a4bf848f8297d89de7b203fcc6678eb5.', STARTKEY => 'ooo', ENDKEY => 'ppp'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0 2024-12-12T06:12:33,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38541 is added to blk_1073741852_1028 (size=45) 2024-12-12T06:12:33,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43493 is added to blk_1073741852_1028 (size=45) 2024-12-12T06:12:33,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42285 is added to blk_1073741852_1028 (size=45) 2024-12-12T06:12:33,779 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ooo,1733983953560.a4bf848f8297d89de7b203fcc6678eb5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-12T06:12:33,780 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing a4bf848f8297d89de7b203fcc6678eb5, disabling compactions & flushes 2024-12-12T06:12:33,780 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ooo,1733983953560.a4bf848f8297d89de7b203fcc6678eb5. 2024-12-12T06:12:33,780 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1733983953560.a4bf848f8297d89de7b203fcc6678eb5. 2024-12-12T06:12:33,780 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1733983953560.a4bf848f8297d89de7b203fcc6678eb5. after waiting 0 ms 2024-12-12T06:12:33,780 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1733983953560.a4bf848f8297d89de7b203fcc6678eb5. 2024-12-12T06:12:33,780 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ooo,1733983953560.a4bf848f8297d89de7b203fcc6678eb5. 2024-12-12T06:12:33,780 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for a4bf848f8297d89de7b203fcc6678eb5: Waiting for close lock at 1733983953780Disabling compacts and flushes for region at 1733983953780Disabling writes for close at 1733983953780Writing region close event to WAL at 1733983953780Closed at 1733983953780 2024-12-12T06:12:33,780 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 1202ee0cad3e428893fcdbc630d4f9b9, NAME => 'test,ppp,1733983953560.1202ee0cad3e428893fcdbc630d4f9b9.', STARTKEY => 'ppp', ENDKEY => 'qqq'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0 2024-12-12T06:12:33,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42285 is added to blk_1073741853_1029 (size=45) 2024-12-12T06:12:33,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38541 is added to blk_1073741853_1029 (size=45) 2024-12-12T06:12:33,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43493 is added to blk_1073741853_1029 (size=45) 2024-12-12T06:12:33,793 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ppp,1733983953560.1202ee0cad3e428893fcdbc630d4f9b9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-12T06:12:33,793 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 1202ee0cad3e428893fcdbc630d4f9b9, disabling compactions & flushes 2024-12-12T06:12:33,794 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ppp,1733983953560.1202ee0cad3e428893fcdbc630d4f9b9. 2024-12-12T06:12:33,794 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1733983953560.1202ee0cad3e428893fcdbc630d4f9b9. 2024-12-12T06:12:33,794 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1733983953560.1202ee0cad3e428893fcdbc630d4f9b9. after waiting 0 ms 2024-12-12T06:12:33,794 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1733983953560.1202ee0cad3e428893fcdbc630d4f9b9. 2024-12-12T06:12:33,794 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ppp,1733983953560.1202ee0cad3e428893fcdbc630d4f9b9. 2024-12-12T06:12:33,794 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 1202ee0cad3e428893fcdbc630d4f9b9: Waiting for close lock at 1733983953793Disabling compacts and flushes for region at 1733983953793Disabling writes for close at 1733983953794 (+1 ms)Writing region close event to WAL at 1733983953794Closed at 1733983953794 2024-12-12T06:12:33,794 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 653572572648db72daea94dd65cfb7f8, NAME => 'test,qqq,1733983953560.653572572648db72daea94dd65cfb7f8.', STARTKEY => 'qqq', ENDKEY => 'rrr'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0 2024-12-12T06:12:33,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42285 is added to blk_1073741854_1030 (size=45) 2024-12-12T06:12:33,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38541 is added to blk_1073741854_1030 (size=45) 2024-12-12T06:12:33,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43493 is added to blk_1073741854_1030 (size=45) 2024-12-12T06:12:33,806 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,qqq,1733983953560.653572572648db72daea94dd65cfb7f8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-12T06:12:33,807 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 653572572648db72daea94dd65cfb7f8, disabling compactions & flushes 2024-12-12T06:12:33,807 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,qqq,1733983953560.653572572648db72daea94dd65cfb7f8. 2024-12-12T06:12:33,807 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1733983953560.653572572648db72daea94dd65cfb7f8. 2024-12-12T06:12:33,807 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1733983953560.653572572648db72daea94dd65cfb7f8. after waiting 0 ms 2024-12-12T06:12:33,807 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1733983953560.653572572648db72daea94dd65cfb7f8. 2024-12-12T06:12:33,807 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,qqq,1733983953560.653572572648db72daea94dd65cfb7f8. 2024-12-12T06:12:33,807 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 653572572648db72daea94dd65cfb7f8: Waiting for close lock at 1733983953807Disabling compacts and flushes for region at 1733983953807Disabling writes for close at 1733983953807Writing region close event to WAL at 1733983953807Closed at 1733983953807 2024-12-12T06:12:33,808 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => f5fe5dc5eb9193cba5339ee508dff867, NAME => 'test,rrr,1733983953560.f5fe5dc5eb9193cba5339ee508dff867.', STARTKEY => 'rrr', ENDKEY => 'sss'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0 2024-12-12T06:12:33,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43493 is added to blk_1073741855_1031 (size=45) 2024-12-12T06:12:33,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38541 is added to blk_1073741855_1031 (size=45) 2024-12-12T06:12:33,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42285 is added to blk_1073741855_1031 (size=45) 2024-12-12T06:12:33,819 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,rrr,1733983953560.f5fe5dc5eb9193cba5339ee508dff867.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-12T06:12:33,819 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing f5fe5dc5eb9193cba5339ee508dff867, disabling compactions & flushes 2024-12-12T06:12:33,819 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,rrr,1733983953560.f5fe5dc5eb9193cba5339ee508dff867. 2024-12-12T06:12:33,819 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1733983953560.f5fe5dc5eb9193cba5339ee508dff867. 2024-12-12T06:12:33,819 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1733983953560.f5fe5dc5eb9193cba5339ee508dff867. after waiting 0 ms 2024-12-12T06:12:33,819 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1733983953560.f5fe5dc5eb9193cba5339ee508dff867. 2024-12-12T06:12:33,819 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,rrr,1733983953560.f5fe5dc5eb9193cba5339ee508dff867. 2024-12-12T06:12:33,819 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for f5fe5dc5eb9193cba5339ee508dff867: Waiting for close lock at 1733983953819Disabling compacts and flushes for region at 1733983953819Disabling writes for close at 1733983953819Writing region close event to WAL at 1733983953819Closed at 1733983953819 2024-12-12T06:12:33,820 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 145c6ba0291376a96e3a37f6b274f63b, NAME => 'test,sss,1733983953560.145c6ba0291376a96e3a37f6b274f63b.', STARTKEY => 'sss', ENDKEY => 'ttt'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0 2024-12-12T06:12:33,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42285 is added to blk_1073741856_1032 (size=45) 2024-12-12T06:12:33,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38541 is added to blk_1073741856_1032 (size=45) 2024-12-12T06:12:33,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43493 is added to blk_1073741856_1032 (size=45) 2024-12-12T06:12:33,831 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,sss,1733983953560.145c6ba0291376a96e3a37f6b274f63b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-12T06:12:33,831 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 145c6ba0291376a96e3a37f6b274f63b, disabling compactions & flushes 2024-12-12T06:12:33,832 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,sss,1733983953560.145c6ba0291376a96e3a37f6b274f63b. 2024-12-12T06:12:33,832 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1733983953560.145c6ba0291376a96e3a37f6b274f63b. 2024-12-12T06:12:33,832 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,sss,1733983953560.145c6ba0291376a96e3a37f6b274f63b. after waiting 0 ms 2024-12-12T06:12:33,832 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,sss,1733983953560.145c6ba0291376a96e3a37f6b274f63b. 2024-12-12T06:12:33,832 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,sss,1733983953560.145c6ba0291376a96e3a37f6b274f63b. 2024-12-12T06:12:33,832 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 145c6ba0291376a96e3a37f6b274f63b: Waiting for close lock at 1733983953831Disabling compacts and flushes for region at 1733983953831Disabling writes for close at 1733983953832 (+1 ms)Writing region close event to WAL at 1733983953832Closed at 1733983953832 2024-12-12T06:12:33,832 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 5162d3c0b3255050cbeed27e4456e5c2, NAME => 'test,ttt,1733983953560.5162d3c0b3255050cbeed27e4456e5c2.', STARTKEY => 'ttt', ENDKEY => 'uuu'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0 2024-12-12T06:12:33,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43493 is added to blk_1073741857_1033 (size=45) 2024-12-12T06:12:33,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38541 is added to blk_1073741857_1033 (size=45) 2024-12-12T06:12:33,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42285 is added to blk_1073741857_1033 (size=45) 2024-12-12T06:12:33,844 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ttt,1733983953560.5162d3c0b3255050cbeed27e4456e5c2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-12T06:12:33,844 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 5162d3c0b3255050cbeed27e4456e5c2, disabling compactions & flushes 2024-12-12T06:12:33,844 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ttt,1733983953560.5162d3c0b3255050cbeed27e4456e5c2. 2024-12-12T06:12:33,844 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1733983953560.5162d3c0b3255050cbeed27e4456e5c2. 2024-12-12T06:12:33,844 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1733983953560.5162d3c0b3255050cbeed27e4456e5c2. after waiting 0 ms 2024-12-12T06:12:33,844 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1733983953560.5162d3c0b3255050cbeed27e4456e5c2. 2024-12-12T06:12:33,844 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ttt,1733983953560.5162d3c0b3255050cbeed27e4456e5c2. 2024-12-12T06:12:33,844 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 5162d3c0b3255050cbeed27e4456e5c2: Waiting for close lock at 1733983953844Disabling compacts and flushes for region at 1733983953844Disabling writes for close at 1733983953844Writing region close event to WAL at 1733983953844Closed at 1733983953844 2024-12-12T06:12:33,845 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => d3ce5f64ef1889a3856db9d8c0f63a72, NAME => 'test,uuu,1733983953560.d3ce5f64ef1889a3856db9d8c0f63a72.', STARTKEY => 'uuu', ENDKEY => 'vvv'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0 2024-12-12T06:12:33,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42285 is added to blk_1073741858_1034 (size=45) 2024-12-12T06:12:33,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43493 is added to blk_1073741858_1034 (size=45) 2024-12-12T06:12:33,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38541 is added to blk_1073741858_1034 (size=45) 2024-12-12T06:12:33,856 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,uuu,1733983953560.d3ce5f64ef1889a3856db9d8c0f63a72.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-12T06:12:33,856 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing d3ce5f64ef1889a3856db9d8c0f63a72, disabling compactions & flushes 2024-12-12T06:12:33,856 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,uuu,1733983953560.d3ce5f64ef1889a3856db9d8c0f63a72. 2024-12-12T06:12:33,856 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1733983953560.d3ce5f64ef1889a3856db9d8c0f63a72. 2024-12-12T06:12:33,856 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1733983953560.d3ce5f64ef1889a3856db9d8c0f63a72. after waiting 0 ms 2024-12-12T06:12:33,856 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1733983953560.d3ce5f64ef1889a3856db9d8c0f63a72. 2024-12-12T06:12:33,856 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,uuu,1733983953560.d3ce5f64ef1889a3856db9d8c0f63a72. 2024-12-12T06:12:33,856 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for d3ce5f64ef1889a3856db9d8c0f63a72: Waiting for close lock at 1733983953856Disabling compacts and flushes for region at 1733983953856Disabling writes for close at 1733983953856Writing region close event to WAL at 1733983953856Closed at 1733983953856 2024-12-12T06:12:33,857 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 60c948f12e02b6ab0066426b6aceea80, NAME => 'test,vvv,1733983953560.60c948f12e02b6ab0066426b6aceea80.', STARTKEY => 'vvv', ENDKEY => 'www'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0 2024-12-12T06:12:33,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43493 is added to blk_1073741859_1035 (size=45) 2024-12-12T06:12:33,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38541 is added to blk_1073741859_1035 (size=45) 2024-12-12T06:12:33,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42285 is added to blk_1073741859_1035 (size=45) 2024-12-12T06:12:33,871 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,vvv,1733983953560.60c948f12e02b6ab0066426b6aceea80.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-12T06:12:33,871 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 60c948f12e02b6ab0066426b6aceea80, disabling compactions & flushes 2024-12-12T06:12:33,871 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,vvv,1733983953560.60c948f12e02b6ab0066426b6aceea80. 2024-12-12T06:12:33,871 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1733983953560.60c948f12e02b6ab0066426b6aceea80. 2024-12-12T06:12:33,871 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1733983953560.60c948f12e02b6ab0066426b6aceea80. after waiting 0 ms 2024-12-12T06:12:33,871 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1733983953560.60c948f12e02b6ab0066426b6aceea80. 2024-12-12T06:12:33,871 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,vvv,1733983953560.60c948f12e02b6ab0066426b6aceea80. 2024-12-12T06:12:33,871 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 60c948f12e02b6ab0066426b6aceea80: Waiting for close lock at 1733983953871Disabling compacts and flushes for region at 1733983953871Disabling writes for close at 1733983953871Writing region close event to WAL at 1733983953871Closed at 1733983953871 2024-12-12T06:12:33,871 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 764fc3b5431e53dfb445323d0bfd39a6, NAME => 'test,www,1733983953560.764fc3b5431e53dfb445323d0bfd39a6.', STARTKEY => 'www', ENDKEY => 'xxx'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0 2024-12-12T06:12:33,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42285 is added to blk_1073741860_1036 (size=45) 2024-12-12T06:12:33,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43493 is added to blk_1073741860_1036 (size=45) 2024-12-12T06:12:33,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38541 is added to blk_1073741860_1036 (size=45) 2024-12-12T06:12:33,881 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,www,1733983953560.764fc3b5431e53dfb445323d0bfd39a6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-12T06:12:33,881 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 764fc3b5431e53dfb445323d0bfd39a6, disabling compactions & flushes 2024-12-12T06:12:33,881 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,www,1733983953560.764fc3b5431e53dfb445323d0bfd39a6. 2024-12-12T06:12:33,881 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1733983953560.764fc3b5431e53dfb445323d0bfd39a6. 2024-12-12T06:12:33,881 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,www,1733983953560.764fc3b5431e53dfb445323d0bfd39a6. after waiting 0 ms 2024-12-12T06:12:33,881 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,www,1733983953560.764fc3b5431e53dfb445323d0bfd39a6. 2024-12-12T06:12:33,881 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,www,1733983953560.764fc3b5431e53dfb445323d0bfd39a6. 2024-12-12T06:12:33,881 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 764fc3b5431e53dfb445323d0bfd39a6: Waiting for close lock at 1733983953881Disabling compacts and flushes for region at 1733983953881Disabling writes for close at 1733983953881Writing region close event to WAL at 1733983953881Closed at 1733983953881 2024-12-12T06:12:33,882 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 269d56dd3a6671f5c9a30401b73ade89, NAME => 'test,xxx,1733983953560.269d56dd3a6671f5c9a30401b73ade89.', STARTKEY => 'xxx', ENDKEY => 'yyy'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0 2024-12-12T06:12:33,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43493 is added to blk_1073741861_1037 (size=45) 2024-12-12T06:12:33,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42285 is added to blk_1073741861_1037 (size=45) 2024-12-12T06:12:33,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38541 is added to blk_1073741861_1037 (size=45) 2024-12-12T06:12:33,892 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,xxx,1733983953560.269d56dd3a6671f5c9a30401b73ade89.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-12T06:12:33,892 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 269d56dd3a6671f5c9a30401b73ade89, disabling compactions & flushes 2024-12-12T06:12:33,892 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,xxx,1733983953560.269d56dd3a6671f5c9a30401b73ade89. 2024-12-12T06:12:33,892 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1733983953560.269d56dd3a6671f5c9a30401b73ade89. 2024-12-12T06:12:33,892 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1733983953560.269d56dd3a6671f5c9a30401b73ade89. after waiting 0 ms 2024-12-12T06:12:33,892 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1733983953560.269d56dd3a6671f5c9a30401b73ade89. 2024-12-12T06:12:33,892 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,xxx,1733983953560.269d56dd3a6671f5c9a30401b73ade89. 2024-12-12T06:12:33,892 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 269d56dd3a6671f5c9a30401b73ade89: Waiting for close lock at 1733983953892Disabling compacts and flushes for region at 1733983953892Disabling writes for close at 1733983953892Writing region close event to WAL at 1733983953892Closed at 1733983953892 2024-12-12T06:12:33,893 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => ae428798a6433b4d48a205ab3d14ba42, NAME => 'test,yyy,1733983953560.ae428798a6433b4d48a205ab3d14ba42.', STARTKEY => 'yyy', ENDKEY => 'zzz'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0 2024-12-12T06:12:33,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38541 is added to blk_1073741862_1038 (size=45) 2024-12-12T06:12:33,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43493 is added to blk_1073741862_1038 (size=45) 2024-12-12T06:12:33,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42285 is added to blk_1073741862_1038 (size=45) 2024-12-12T06:12:33,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45209 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-12T06:12:33,911 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,yyy,1733983953560.ae428798a6433b4d48a205ab3d14ba42.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-12T06:12:33,911 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing ae428798a6433b4d48a205ab3d14ba42, disabling compactions & flushes 2024-12-12T06:12:33,911 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,yyy,1733983953560.ae428798a6433b4d48a205ab3d14ba42. 2024-12-12T06:12:33,912 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1733983953560.ae428798a6433b4d48a205ab3d14ba42. 2024-12-12T06:12:33,912 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1733983953560.ae428798a6433b4d48a205ab3d14ba42. after waiting 0 ms 2024-12-12T06:12:33,912 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1733983953560.ae428798a6433b4d48a205ab3d14ba42. 2024-12-12T06:12:33,912 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,yyy,1733983953560.ae428798a6433b4d48a205ab3d14ba42. 2024-12-12T06:12:33,912 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for ae428798a6433b4d48a205ab3d14ba42: Waiting for close lock at 1733983953911Disabling compacts and flushes for region at 1733983953911Disabling writes for close at 1733983953912 (+1 ms)Writing region close event to WAL at 1733983953912Closed at 1733983953912 2024-12-12T06:12:33,912 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 269eed2924060312949b51c10150f409, NAME => 'test,zzz,1733983953560.269eed2924060312949b51c10150f409.', STARTKEY => 'zzz', ENDKEY => ''}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0 2024-12-12T06:12:33,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38541 is added to blk_1073741863_1039 (size=42) 2024-12-12T06:12:33,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42285 is added to blk_1073741863_1039 (size=42) 2024-12-12T06:12:33,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43493 is added to blk_1073741863_1039 (size=42) 2024-12-12T06:12:33,923 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,zzz,1733983953560.269eed2924060312949b51c10150f409.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-12T06:12:33,923 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 269eed2924060312949b51c10150f409, disabling compactions & flushes 2024-12-12T06:12:33,923 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,zzz,1733983953560.269eed2924060312949b51c10150f409. 2024-12-12T06:12:33,923 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1733983953560.269eed2924060312949b51c10150f409. 2024-12-12T06:12:33,923 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1733983953560.269eed2924060312949b51c10150f409. after waiting 0 ms 2024-12-12T06:12:33,924 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1733983953560.269eed2924060312949b51c10150f409. 2024-12-12T06:12:33,924 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,zzz,1733983953560.269eed2924060312949b51c10150f409. 2024-12-12T06:12:33,924 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 269eed2924060312949b51c10150f409: Waiting for close lock at 1733983953923Disabling compacts and flushes for region at 1733983953923Disabling writes for close at 1733983953923Writing region close event to WAL at 1733983953924 (+1 ms)Closed at 1733983953924 2024-12-12T06:12:34,146 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,kkk,1733983953560.782c52424de7d71cb43b8fbfe8e89f7f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-12T06:12:34,146 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,lll,1733983953560.5d34bdb8a1414fb25a475fb97fc440b4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-12T06:12:34,146 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 5d34bdb8a1414fb25a475fb97fc440b4, disabling compactions & flushes 2024-12-12T06:12:34,146 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 782c52424de7d71cb43b8fbfe8e89f7f, disabling compactions & flushes 2024-12-12T06:12:34,146 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,lll,1733983953560.5d34bdb8a1414fb25a475fb97fc440b4. 2024-12-12T06:12:34,146 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,kkk,1733983953560.782c52424de7d71cb43b8fbfe8e89f7f. 2024-12-12T06:12:34,146 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1733983953560.5d34bdb8a1414fb25a475fb97fc440b4. 2024-12-12T06:12:34,146 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1733983953560.782c52424de7d71cb43b8fbfe8e89f7f. 2024-12-12T06:12:34,146 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,lll,1733983953560.5d34bdb8a1414fb25a475fb97fc440b4. after waiting 0 ms 2024-12-12T06:12:34,146 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1733983953560.782c52424de7d71cb43b8fbfe8e89f7f. after waiting 0 ms 2024-12-12T06:12:34,146 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,lll,1733983953560.5d34bdb8a1414fb25a475fb97fc440b4. 2024-12-12T06:12:34,146 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1733983953560.782c52424de7d71cb43b8fbfe8e89f7f. 2024-12-12T06:12:34,146 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,lll,1733983953560.5d34bdb8a1414fb25a475fb97fc440b4. 2024-12-12T06:12:34,146 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,kkk,1733983953560.782c52424de7d71cb43b8fbfe8e89f7f. 2024-12-12T06:12:34,147 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 782c52424de7d71cb43b8fbfe8e89f7f: Waiting for close lock at 1733983954146Disabling compacts and flushes for region at 1733983954146Disabling writes for close at 1733983954146Writing region close event to WAL at 1733983954146Closed at 1733983954146 2024-12-12T06:12:34,147 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 5d34bdb8a1414fb25a475fb97fc440b4: Waiting for close lock at 1733983954146Disabling compacts and flushes for region at 1733983954146Disabling writes for close at 1733983954146Writing region close event to WAL at 1733983954146Closed at 1733983954146 2024-12-12T06:12:34,150 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ADD_TO_META 2024-12-12T06:12:34,156 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ccc,1733983953560.2eeaafb9a1964f4c6899fa5dde05a463.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733983954150"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733983954150"}]},"ts":"1733983954150"} 2024-12-12T06:12:34,156 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,,1733983953560.8545c1fe97252abe3447dcfaebd5cfa5.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1733983954150"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733983954150"}]},"ts":"1733983954150"} 2024-12-12T06:12:34,157 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,bbb,1733983953560.16838c5df0ab4e3938ad84e9a080def9.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733983954150"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733983954150"}]},"ts":"1733983954150"} 2024-12-12T06:12:34,157 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ddd,1733983953560.c98bb859cc9e047b211ddb4c5b967ce5.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733983954150"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733983954150"}]},"ts":"1733983954150"} 2024-12-12T06:12:34,157 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,fff,1733983953560.4584d26a35c36968de829b34ac08fb5d.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733983954150"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733983954150"}]},"ts":"1733983954150"} 2024-12-12T06:12:34,157 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,eee,1733983953560.b935ee7eeea02b161c0ebc0d05527e91.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733983954150"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733983954150"}]},"ts":"1733983954150"} 2024-12-12T06:12:34,157 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ggg,1733983953560.b717f65da42dacb67935407f094e8467.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733983954150"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733983954150"}]},"ts":"1733983954150"} 2024-12-12T06:12:34,157 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,iii,1733983953560.57995ad3e527ada56b846564833341d1.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733983954150"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733983954150"}]},"ts":"1733983954150"} 2024-12-12T06:12:34,158 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,hhh,1733983953560.f2b6a6b11f14670aca9dccceb20d9630.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733983954150"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733983954150"}]},"ts":"1733983954150"} 2024-12-12T06:12:34,158 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,jjj,1733983953560.c77db308a7c1c57fe9ef945e17e28127.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733983954150"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733983954150"}]},"ts":"1733983954150"} 2024-12-12T06:12:34,158 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,mmm,1733983953560.d86220c39cb75f305e7ed4d4c9f1191c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733983954150"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733983954150"}]},"ts":"1733983954150"} 2024-12-12T06:12:34,158 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,nnn,1733983953560.be4cb5da32eefc543e09e80b426b6938.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733983954150"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733983954150"}]},"ts":"1733983954150"} 2024-12-12T06:12:34,158 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ooo,1733983953560.a4bf848f8297d89de7b203fcc6678eb5.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733983954150"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733983954150"}]},"ts":"1733983954150"} 2024-12-12T06:12:34,158 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ppp,1733983953560.1202ee0cad3e428893fcdbc630d4f9b9.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733983954150"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733983954150"}]},"ts":"1733983954150"} 2024-12-12T06:12:34,158 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,qqq,1733983953560.653572572648db72daea94dd65cfb7f8.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733983954150"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733983954150"}]},"ts":"1733983954150"} 2024-12-12T06:12:34,159 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,rrr,1733983953560.f5fe5dc5eb9193cba5339ee508dff867.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733983954150"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733983954150"}]},"ts":"1733983954150"} 2024-12-12T06:12:34,159 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,sss,1733983953560.145c6ba0291376a96e3a37f6b274f63b.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733983954150"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733983954150"}]},"ts":"1733983954150"} 2024-12-12T06:12:34,159 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ttt,1733983953560.5162d3c0b3255050cbeed27e4456e5c2.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733983954150"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733983954150"}]},"ts":"1733983954150"} 2024-12-12T06:12:34,159 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,uuu,1733983953560.d3ce5f64ef1889a3856db9d8c0f63a72.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733983954150"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733983954150"}]},"ts":"1733983954150"} 2024-12-12T06:12:34,159 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,vvv,1733983953560.60c948f12e02b6ab0066426b6aceea80.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733983954150"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733983954150"}]},"ts":"1733983954150"} 2024-12-12T06:12:34,159 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,www,1733983953560.764fc3b5431e53dfb445323d0bfd39a6.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733983954150"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733983954150"}]},"ts":"1733983954150"} 2024-12-12T06:12:34,159 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,xxx,1733983953560.269d56dd3a6671f5c9a30401b73ade89.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733983954150"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733983954150"}]},"ts":"1733983954150"} 2024-12-12T06:12:34,160 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,yyy,1733983953560.ae428798a6433b4d48a205ab3d14ba42.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733983954150"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733983954150"}]},"ts":"1733983954150"} 2024-12-12T06:12:34,160 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,zzz,1733983953560.269eed2924060312949b51c10150f409.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1733983954150"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733983954150"}]},"ts":"1733983954150"} 2024-12-12T06:12:34,160 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,kkk,1733983953560.782c52424de7d71cb43b8fbfe8e89f7f.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733983954150"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733983954150"}]},"ts":"1733983954150"} 2024-12-12T06:12:34,160 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,lll,1733983953560.5d34bdb8a1414fb25a475fb97fc440b4.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733983954150"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733983954150"}]},"ts":"1733983954150"} 2024-12-12T06:12:34,196 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 26 regions to meta. 2024-12-12T06:12:34,198 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-12T06:12:34,200 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733983954198"}]},"ts":"1733983954198"} 2024-12-12T06:12:34,204 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLING in hbase:meta 2024-12-12T06:12:34,205 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {ffbfd3107920=0} racks are {/default-rack=0} 2024-12-12T06:12:34,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45209 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-12T06:12:34,221 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-12T06:12:34,221 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-12T06:12:34,221 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-12T06:12:34,221 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-12T06:12:34,221 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-12T06:12:34,221 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-12T06:12:34,221 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-12T06:12:34,221 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-12T06:12:34,221 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-12T06:12:34,222 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-12T06:12:34,224 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8545c1fe97252abe3447dcfaebd5cfa5, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=16838c5df0ab4e3938ad84e9a080def9, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2eeaafb9a1964f4c6899fa5dde05a463, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c98bb859cc9e047b211ddb4c5b967ce5, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b935ee7eeea02b161c0ebc0d05527e91, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4584d26a35c36968de829b34ac08fb5d, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b717f65da42dacb67935407f094e8467, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f2b6a6b11f14670aca9dccceb20d9630, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c77db308a7c1c57fe9ef945e17e28127, ASSIGN}, {pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=782c52424de7d71cb43b8fbfe8e89f7f, ASSIGN}, {pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN}, {pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d86220c39cb75f305e7ed4d4c9f1191c, ASSIGN}, {pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=be4cb5da32eefc543e09e80b426b6938, ASSIGN}, {pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a4bf848f8297d89de7b203fcc6678eb5, ASSIGN}, {pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1202ee0cad3e428893fcdbc630d4f9b9, ASSIGN}, {pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=653572572648db72daea94dd65cfb7f8, ASSIGN}, {pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f5fe5dc5eb9193cba5339ee508dff867, ASSIGN}, {pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN}, {pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5162d3c0b3255050cbeed27e4456e5c2, ASSIGN}, {pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d3ce5f64ef1889a3856db9d8c0f63a72, ASSIGN}, {pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=60c948f12e02b6ab0066426b6aceea80, ASSIGN}, {pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=764fc3b5431e53dfb445323d0bfd39a6, ASSIGN}, {pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=269d56dd3a6671f5c9a30401b73ade89, ASSIGN}, {pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ae428798a6433b4d48a205ab3d14ba42, ASSIGN}, {pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=269eed2924060312949b51c10150f409, ASSIGN}] 2024-12-12T06:12:34,232 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=269eed2924060312949b51c10150f409, ASSIGN 2024-12-12T06:12:34,234 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=269eed2924060312949b51c10150f409, ASSIGN; state=OFFLINE, location=ffbfd3107920,42275,1733983951224; forceNewPlan=false, retain=false 2024-12-12T06:12:34,235 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ae428798a6433b4d48a205ab3d14ba42, ASSIGN 2024-12-12T06:12:34,236 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ae428798a6433b4d48a205ab3d14ba42, ASSIGN; state=OFFLINE, location=ffbfd3107920,33455,1733983951151; forceNewPlan=false, retain=false 2024-12-12T06:12:34,238 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=269d56dd3a6671f5c9a30401b73ade89, ASSIGN 2024-12-12T06:12:34,239 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=269d56dd3a6671f5c9a30401b73ade89, ASSIGN; state=OFFLINE, location=ffbfd3107920,46199,1733983951048; forceNewPlan=false, retain=false 2024-12-12T06:12:34,241 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=764fc3b5431e53dfb445323d0bfd39a6, ASSIGN 2024-12-12T06:12:34,242 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=764fc3b5431e53dfb445323d0bfd39a6, ASSIGN; state=OFFLINE, location=ffbfd3107920,42275,1733983951224; forceNewPlan=false, retain=false 2024-12-12T06:12:34,243 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=60c948f12e02b6ab0066426b6aceea80, ASSIGN 2024-12-12T06:12:34,244 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=60c948f12e02b6ab0066426b6aceea80, ASSIGN; state=OFFLINE, location=ffbfd3107920,33455,1733983951151; forceNewPlan=false, retain=false 2024-12-12T06:12:34,246 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d3ce5f64ef1889a3856db9d8c0f63a72, ASSIGN 2024-12-12T06:12:34,247 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d3ce5f64ef1889a3856db9d8c0f63a72, ASSIGN; state=OFFLINE, location=ffbfd3107920,46199,1733983951048; forceNewPlan=false, retain=false 2024-12-12T06:12:34,248 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5162d3c0b3255050cbeed27e4456e5c2, ASSIGN 2024-12-12T06:12:34,249 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=5162d3c0b3255050cbeed27e4456e5c2, ASSIGN; state=OFFLINE, location=ffbfd3107920,42275,1733983951224; forceNewPlan=false, retain=false 2024-12-12T06:12:34,251 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN 2024-12-12T06:12:34,252 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN; state=OFFLINE, location=ffbfd3107920,33455,1733983951151; forceNewPlan=false, retain=false 2024-12-12T06:12:34,253 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f5fe5dc5eb9193cba5339ee508dff867, ASSIGN 2024-12-12T06:12:34,255 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f5fe5dc5eb9193cba5339ee508dff867, ASSIGN; state=OFFLINE, location=ffbfd3107920,46199,1733983951048; forceNewPlan=false, retain=false 2024-12-12T06:12:34,256 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=653572572648db72daea94dd65cfb7f8, ASSIGN 2024-12-12T06:12:34,257 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=653572572648db72daea94dd65cfb7f8, ASSIGN; state=OFFLINE, location=ffbfd3107920,42275,1733983951224; forceNewPlan=false, retain=false 2024-12-12T06:12:34,259 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1202ee0cad3e428893fcdbc630d4f9b9, ASSIGN 2024-12-12T06:12:34,260 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=1202ee0cad3e428893fcdbc630d4f9b9, ASSIGN; state=OFFLINE, location=ffbfd3107920,33455,1733983951151; forceNewPlan=false, retain=false 2024-12-12T06:12:34,261 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a4bf848f8297d89de7b203fcc6678eb5, ASSIGN 2024-12-12T06:12:34,263 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a4bf848f8297d89de7b203fcc6678eb5, ASSIGN; state=OFFLINE, location=ffbfd3107920,46199,1733983951048; forceNewPlan=false, retain=false 2024-12-12T06:12:34,264 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=be4cb5da32eefc543e09e80b426b6938, ASSIGN 2024-12-12T06:12:34,265 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=be4cb5da32eefc543e09e80b426b6938, ASSIGN; state=OFFLINE, location=ffbfd3107920,42275,1733983951224; forceNewPlan=false, retain=false 2024-12-12T06:12:34,266 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d86220c39cb75f305e7ed4d4c9f1191c, ASSIGN 2024-12-12T06:12:34,267 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d86220c39cb75f305e7ed4d4c9f1191c, ASSIGN; state=OFFLINE, location=ffbfd3107920,33455,1733983951151; forceNewPlan=false, retain=false 2024-12-12T06:12:34,268 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN 2024-12-12T06:12:34,270 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN; state=OFFLINE, location=ffbfd3107920,33455,1733983951151; forceNewPlan=false, retain=false 2024-12-12T06:12:34,271 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=782c52424de7d71cb43b8fbfe8e89f7f, ASSIGN 2024-12-12T06:12:34,272 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=782c52424de7d71cb43b8fbfe8e89f7f, ASSIGN; state=OFFLINE, location=ffbfd3107920,46199,1733983951048; forceNewPlan=false, retain=false 2024-12-12T06:12:34,273 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c77db308a7c1c57fe9ef945e17e28127, ASSIGN 2024-12-12T06:12:34,274 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c77db308a7c1c57fe9ef945e17e28127, ASSIGN; state=OFFLINE, location=ffbfd3107920,46199,1733983951048; forceNewPlan=false, retain=false 2024-12-12T06:12:34,275 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN 2024-12-12T06:12:34,276 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN; state=OFFLINE, location=ffbfd3107920,33455,1733983951151; forceNewPlan=false, retain=false 2024-12-12T06:12:34,278 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f2b6a6b11f14670aca9dccceb20d9630, ASSIGN 2024-12-12T06:12:34,279 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f2b6a6b11f14670aca9dccceb20d9630, ASSIGN; state=OFFLINE, location=ffbfd3107920,42275,1733983951224; forceNewPlan=false, retain=false 2024-12-12T06:12:34,280 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b717f65da42dacb67935407f094e8467, ASSIGN 2024-12-12T06:12:34,281 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=b717f65da42dacb67935407f094e8467, ASSIGN; state=OFFLINE, location=ffbfd3107920,46199,1733983951048; forceNewPlan=false, retain=false 2024-12-12T06:12:34,282 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4584d26a35c36968de829b34ac08fb5d, ASSIGN 2024-12-12T06:12:34,283 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4584d26a35c36968de829b34ac08fb5d, ASSIGN; state=OFFLINE, location=ffbfd3107920,33455,1733983951151; forceNewPlan=false, retain=false 2024-12-12T06:12:34,284 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b935ee7eeea02b161c0ebc0d05527e91, ASSIGN 2024-12-12T06:12:34,285 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=b935ee7eeea02b161c0ebc0d05527e91, ASSIGN; state=OFFLINE, location=ffbfd3107920,42275,1733983951224; forceNewPlan=false, retain=false 2024-12-12T06:12:34,286 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c98bb859cc9e047b211ddb4c5b967ce5, ASSIGN 2024-12-12T06:12:34,287 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c98bb859cc9e047b211ddb4c5b967ce5, ASSIGN; state=OFFLINE, location=ffbfd3107920,46199,1733983951048; forceNewPlan=false, retain=false 2024-12-12T06:12:34,288 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2eeaafb9a1964f4c6899fa5dde05a463, ASSIGN 2024-12-12T06:12:34,289 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=2eeaafb9a1964f4c6899fa5dde05a463, ASSIGN; state=OFFLINE, location=ffbfd3107920,46199,1733983951048; forceNewPlan=false, retain=false 2024-12-12T06:12:34,290 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=16838c5df0ab4e3938ad84e9a080def9, ASSIGN 2024-12-12T06:12:34,292 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=16838c5df0ab4e3938ad84e9a080def9, ASSIGN; state=OFFLINE, location=ffbfd3107920,42275,1733983951224; forceNewPlan=false, retain=false 2024-12-12T06:12:34,293 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8545c1fe97252abe3447dcfaebd5cfa5, ASSIGN 2024-12-12T06:12:34,294 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8545c1fe97252abe3447dcfaebd5cfa5, ASSIGN; state=OFFLINE, location=ffbfd3107920,33455,1733983951151; forceNewPlan=false, retain=false 2024-12-12T06:12:34,389 INFO [ffbfd3107920:45209 {}] balancer.BaseLoadBalancer(388): Reassigned 26 regions. 26 retained the pre-restart assignment. 2024-12-12T06:12:34,391 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=269d56dd3a6671f5c9a30401b73ade89, regionState=OPENING, regionLocation=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:34,393 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=d3ce5f64ef1889a3856db9d8c0f63a72, regionState=OPENING, regionLocation=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:34,393 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=f5fe5dc5eb9193cba5339ee508dff867, regionState=OPENING, regionLocation=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:34,394 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=a4bf848f8297d89de7b203fcc6678eb5, regionState=OPENING, regionLocation=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:34,395 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=782c52424de7d71cb43b8fbfe8e89f7f, regionState=OPENING, regionLocation=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:34,395 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=269d56dd3a6671f5c9a30401b73ade89, ASSIGN because future has completed 2024-12-12T06:12:34,396 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=c77db308a7c1c57fe9ef945e17e28127, regionState=OPENING, regionLocation=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:34,396 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=b717f65da42dacb67935407f094e8467, regionState=OPENING, regionLocation=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:34,397 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 269d56dd3a6671f5c9a30401b73ade89, server=ffbfd3107920,46199,1733983951048}] 2024-12-12T06:12:34,398 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d3ce5f64ef1889a3856db9d8c0f63a72, ASSIGN because future has completed 2024-12-12T06:12:34,400 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f5fe5dc5eb9193cba5339ee508dff867, ASSIGN because future has completed 2024-12-12T06:12:34,402 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a4bf848f8297d89de7b203fcc6678eb5, ASSIGN because future has completed 2024-12-12T06:12:34,404 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=782c52424de7d71cb43b8fbfe8e89f7f, ASSIGN because future has completed 2024-12-12T06:12:34,406 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 782c52424de7d71cb43b8fbfe8e89f7f, server=ffbfd3107920,46199,1733983951048}] 2024-12-12T06:12:34,406 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c77db308a7c1c57fe9ef945e17e28127, ASSIGN because future has completed 2024-12-12T06:12:34,409 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=b717f65da42dacb67935407f094e8467, ASSIGN because future has completed 2024-12-12T06:12:34,418 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure b717f65da42dacb67935407f094e8467, server=ffbfd3107920,46199,1733983951048}] 2024-12-12T06:12:34,424 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure c77db308a7c1c57fe9ef945e17e28127, server=ffbfd3107920,46199,1733983951048}] 2024-12-12T06:12:34,430 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure a4bf848f8297d89de7b203fcc6678eb5, server=ffbfd3107920,46199,1733983951048}] 2024-12-12T06:12:34,435 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5fe5dc5eb9193cba5339ee508dff867, server=ffbfd3107920,46199,1733983951048}] 2024-12-12T06:12:34,440 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure d3ce5f64ef1889a3856db9d8c0f63a72, server=ffbfd3107920,46199,1733983951048}] 2024-12-12T06:12:34,447 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=c98bb859cc9e047b211ddb4c5b967ce5, regionState=OPENING, regionLocation=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:34,448 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=2eeaafb9a1964f4c6899fa5dde05a463, regionState=OPENING, regionLocation=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:34,449 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=269eed2924060312949b51c10150f409, regionState=OPENING, regionLocation=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:34,449 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=764fc3b5431e53dfb445323d0bfd39a6, regionState=OPENING, regionLocation=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:34,450 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=5162d3c0b3255050cbeed27e4456e5c2, regionState=OPENING, regionLocation=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:34,450 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c98bb859cc9e047b211ddb4c5b967ce5, ASSIGN because future has completed 2024-12-12T06:12:34,451 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=653572572648db72daea94dd65cfb7f8, regionState=OPENING, regionLocation=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:34,451 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure c98bb859cc9e047b211ddb4c5b967ce5, server=ffbfd3107920,46199,1733983951048}] 2024-12-12T06:12:34,452 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=2eeaafb9a1964f4c6899fa5dde05a463, ASSIGN because future has completed 2024-12-12T06:12:34,453 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=269eed2924060312949b51c10150f409, ASSIGN because future has completed 2024-12-12T06:12:34,455 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=764fc3b5431e53dfb445323d0bfd39a6, ASSIGN because future has completed 2024-12-12T06:12:34,456 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=5162d3c0b3255050cbeed27e4456e5c2, ASSIGN because future has completed 2024-12-12T06:12:34,457 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5162d3c0b3255050cbeed27e4456e5c2, server=ffbfd3107920,42275,1733983951224}] 2024-12-12T06:12:34,458 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=653572572648db72daea94dd65cfb7f8, ASSIGN because future has completed 2024-12-12T06:12:34,461 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 653572572648db72daea94dd65cfb7f8, server=ffbfd3107920,42275,1733983951224}] 2024-12-12T06:12:34,465 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 764fc3b5431e53dfb445323d0bfd39a6, server=ffbfd3107920,42275,1733983951224}] 2024-12-12T06:12:34,469 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 269eed2924060312949b51c10150f409, server=ffbfd3107920,42275,1733983951224}] 2024-12-12T06:12:34,473 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2eeaafb9a1964f4c6899fa5dde05a463, server=ffbfd3107920,46199,1733983951048}] 2024-12-12T06:12:34,477 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=be4cb5da32eefc543e09e80b426b6938, regionState=OPENING, regionLocation=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:34,477 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=f2b6a6b11f14670aca9dccceb20d9630, regionState=OPENING, regionLocation=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:34,478 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=b935ee7eeea02b161c0ebc0d05527e91, regionState=OPENING, regionLocation=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:34,478 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=16838c5df0ab4e3938ad84e9a080def9, regionState=OPENING, regionLocation=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:34,479 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=ae428798a6433b4d48a205ab3d14ba42, regionState=OPENING, regionLocation=ffbfd3107920,33455,1733983951151 2024-12-12T06:12:34,480 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=be4cb5da32eefc543e09e80b426b6938, ASSIGN because future has completed 2024-12-12T06:12:34,480 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=60c948f12e02b6ab0066426b6aceea80, regionState=OPENING, regionLocation=ffbfd3107920,33455,1733983951151 2024-12-12T06:12:34,481 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure be4cb5da32eefc543e09e80b426b6938, server=ffbfd3107920,42275,1733983951224}] 2024-12-12T06:12:34,481 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f2b6a6b11f14670aca9dccceb20d9630, ASSIGN because future has completed 2024-12-12T06:12:34,483 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=b935ee7eeea02b161c0ebc0d05527e91, ASSIGN because future has completed 2024-12-12T06:12:34,484 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=16838c5df0ab4e3938ad84e9a080def9, ASSIGN because future has completed 2024-12-12T06:12:34,486 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ae428798a6433b4d48a205ab3d14ba42, ASSIGN because future has completed 2024-12-12T06:12:34,487 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure ae428798a6433b4d48a205ab3d14ba42, server=ffbfd3107920,33455,1733983951151}] 2024-12-12T06:12:34,487 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=60c948f12e02b6ab0066426b6aceea80, ASSIGN because future has completed 2024-12-12T06:12:34,491 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 60c948f12e02b6ab0066426b6aceea80, server=ffbfd3107920,33455,1733983951151}] 2024-12-12T06:12:34,496 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 16838c5df0ab4e3938ad84e9a080def9, server=ffbfd3107920,42275,1733983951224}] 2024-12-12T06:12:34,501 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure b935ee7eeea02b161c0ebc0d05527e91, server=ffbfd3107920,42275,1733983951224}] 2024-12-12T06:12:34,505 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure f2b6a6b11f14670aca9dccceb20d9630, server=ffbfd3107920,42275,1733983951224}] 2024-12-12T06:12:34,509 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=145c6ba0291376a96e3a37f6b274f63b, regionState=OPENING, regionLocation=ffbfd3107920,33455,1733983951151 2024-12-12T06:12:34,510 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=1202ee0cad3e428893fcdbc630d4f9b9, regionState=OPENING, regionLocation=ffbfd3107920,33455,1733983951151 2024-12-12T06:12:34,511 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=d86220c39cb75f305e7ed4d4c9f1191c, regionState=OPENING, regionLocation=ffbfd3107920,33455,1733983951151 2024-12-12T06:12:34,511 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=5d34bdb8a1414fb25a475fb97fc440b4, regionState=OPENING, regionLocation=ffbfd3107920,33455,1733983951151 2024-12-12T06:12:34,512 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=57995ad3e527ada56b846564833341d1, regionState=OPENING, regionLocation=ffbfd3107920,33455,1733983951151 2024-12-12T06:12:34,512 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=4584d26a35c36968de829b34ac08fb5d, regionState=OPENING, regionLocation=ffbfd3107920,33455,1733983951151 2024-12-12T06:12:34,513 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN because future has completed 2024-12-12T06:12:34,513 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 145c6ba0291376a96e3a37f6b274f63b, server=ffbfd3107920,33455,1733983951151}] 2024-12-12T06:12:34,514 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=1202ee0cad3e428893fcdbc630d4f9b9, ASSIGN because future has completed 2024-12-12T06:12:34,515 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d86220c39cb75f305e7ed4d4c9f1191c, ASSIGN because future has completed 2024-12-12T06:12:34,516 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN because future has completed 2024-12-12T06:12:34,518 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN because future has completed 2024-12-12T06:12:34,519 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 57995ad3e527ada56b846564833341d1, server=ffbfd3107920,33455,1733983951151}] 2024-12-12T06:12:34,519 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4584d26a35c36968de829b34ac08fb5d, ASSIGN because future has completed 2024-12-12T06:12:34,523 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4584d26a35c36968de829b34ac08fb5d, server=ffbfd3107920,33455,1733983951151}] 2024-12-12T06:12:34,527 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,33455,1733983951151}] 2024-12-12T06:12:34,531 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure d86220c39cb75f305e7ed4d4c9f1191c, server=ffbfd3107920,33455,1733983951151}] 2024-12-12T06:12:34,535 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1202ee0cad3e428893fcdbc630d4f9b9, server=ffbfd3107920,33455,1733983951151}] 2024-12-12T06:12:34,539 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=8545c1fe97252abe3447dcfaebd5cfa5, regionState=OPENING, regionLocation=ffbfd3107920,33455,1733983951151 2024-12-12T06:12:34,541 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8545c1fe97252abe3447dcfaebd5cfa5, ASSIGN because future has completed 2024-12-12T06:12:34,542 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8545c1fe97252abe3447dcfaebd5cfa5, server=ffbfd3107920,33455,1733983951151}] 2024-12-12T06:12:34,553 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-12T06:12:34,556 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57137, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-12T06:12:34,562 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open test,jjj,1733983953560.c77db308a7c1c57fe9ef945e17e28127. 2024-12-12T06:12:34,562 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => c77db308a7c1c57fe9ef945e17e28127, NAME => 'test,jjj,1733983953560.c77db308a7c1c57fe9ef945e17e28127.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-12-12T06:12:34,562 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c77db308a7c1c57fe9ef945e17e28127 2024-12-12T06:12:34,562 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated test,jjj,1733983953560.c77db308a7c1c57fe9ef945e17e28127.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-12T06:12:34,562 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for c77db308a7c1c57fe9ef945e17e28127 2024-12-12T06:12:34,563 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for c77db308a7c1c57fe9ef945e17e28127 2024-12-12T06:12:34,565 INFO [StoreOpener-c77db308a7c1c57fe9ef945e17e28127-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c77db308a7c1c57fe9ef945e17e28127 2024-12-12T06:12:34,567 INFO [StoreOpener-c77db308a7c1c57fe9ef945e17e28127-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c77db308a7c1c57fe9ef945e17e28127 columnFamilyName family 2024-12-12T06:12:34,567 DEBUG [StoreOpener-c77db308a7c1c57fe9ef945e17e28127-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-12T06:12:34,568 INFO [StoreOpener-c77db308a7c1c57fe9ef945e17e28127-1 {}] regionserver.HStore(327): Store=c77db308a7c1c57fe9ef945e17e28127/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-12T06:12:34,568 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for c77db308a7c1c57fe9ef945e17e28127 2024-12-12T06:12:34,570 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/c77db308a7c1c57fe9ef945e17e28127 2024-12-12T06:12:34,571 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/c77db308a7c1c57fe9ef945e17e28127 2024-12-12T06:12:34,571 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for c77db308a7c1c57fe9ef945e17e28127 2024-12-12T06:12:34,571 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for c77db308a7c1c57fe9ef945e17e28127 2024-12-12T06:12:34,574 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for c77db308a7c1c57fe9ef945e17e28127 2024-12-12T06:12:34,577 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/c77db308a7c1c57fe9ef945e17e28127/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-12T06:12:34,578 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened c77db308a7c1c57fe9ef945e17e28127; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68659822, jitterRate=0.023111075162887573}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-12T06:12:34,578 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c77db308a7c1c57fe9ef945e17e28127 2024-12-12T06:12:34,579 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for c77db308a7c1c57fe9ef945e17e28127: Running coprocessor pre-open hook at 1733983954563Writing region info on filesystem at 1733983954563Initializing all the Stores at 1733983954564 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733983954565 (+1 ms)Cleaning up temporary data from old regions at 1733983954571 (+6 ms)Running coprocessor post-open hooks at 1733983954578 (+7 ms)Region opened successfully at 1733983954579 (+1 ms) 2024-12-12T06:12:34,581 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for test,jjj,1733983953560.c77db308a7c1c57fe9ef945e17e28127., pid=34, masterSystemTime=1733983954553 2024-12-12T06:12:34,585 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for test,jjj,1733983953560.c77db308a7c1c57fe9ef945e17e28127. 2024-12-12T06:12:34,585 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened test,jjj,1733983953560.c77db308a7c1c57fe9ef945e17e28127. 2024-12-12T06:12:34,585 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open test,rrr,1733983953560.f5fe5dc5eb9193cba5339ee508dff867. 2024-12-12T06:12:34,586 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => f5fe5dc5eb9193cba5339ee508dff867, NAME => 'test,rrr,1733983953560.f5fe5dc5eb9193cba5339ee508dff867.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-12-12T06:12:34,586 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=c77db308a7c1c57fe9ef945e17e28127, regionState=OPEN, openSeqNum=2, regionLocation=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:34,586 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f5fe5dc5eb9193cba5339ee508dff867 2024-12-12T06:12:34,586 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated test,rrr,1733983953560.f5fe5dc5eb9193cba5339ee508dff867.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-12T06:12:34,586 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for f5fe5dc5eb9193cba5339ee508dff867 2024-12-12T06:12:34,587 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for f5fe5dc5eb9193cba5339ee508dff867 2024-12-12T06:12:34,589 INFO [StoreOpener-f5fe5dc5eb9193cba5339ee508dff867-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f5fe5dc5eb9193cba5339ee508dff867 2024-12-12T06:12:34,590 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure c77db308a7c1c57fe9ef945e17e28127, server=ffbfd3107920,46199,1733983951048 because future has completed 2024-12-12T06:12:34,592 INFO [StoreOpener-f5fe5dc5eb9193cba5339ee508dff867-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f5fe5dc5eb9193cba5339ee508dff867 columnFamilyName family 2024-12-12T06:12:34,592 DEBUG [StoreOpener-f5fe5dc5eb9193cba5339ee508dff867-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-12T06:12:34,593 INFO [StoreOpener-f5fe5dc5eb9193cba5339ee508dff867-1 {}] regionserver.HStore(327): Store=f5fe5dc5eb9193cba5339ee508dff867/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-12T06:12:34,593 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for f5fe5dc5eb9193cba5339ee508dff867 2024-12-12T06:12:34,594 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/f5fe5dc5eb9193cba5339ee508dff867 2024-12-12T06:12:34,595 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=14 2024-12-12T06:12:34,595 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure c77db308a7c1c57fe9ef945e17e28127, server=ffbfd3107920,46199,1733983951048 in 168 msec 2024-12-12T06:12:34,595 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/f5fe5dc5eb9193cba5339ee508dff867 2024-12-12T06:12:34,596 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for f5fe5dc5eb9193cba5339ee508dff867 2024-12-12T06:12:34,596 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for f5fe5dc5eb9193cba5339ee508dff867 2024-12-12T06:12:34,597 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=c77db308a7c1c57fe9ef945e17e28127, ASSIGN in 371 msec 2024-12-12T06:12:34,598 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for f5fe5dc5eb9193cba5339ee508dff867 2024-12-12T06:12:34,601 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/f5fe5dc5eb9193cba5339ee508dff867/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-12T06:12:34,601 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened f5fe5dc5eb9193cba5339ee508dff867; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63127177, jitterRate=-0.05933175981044769}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-12T06:12:34,602 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f5fe5dc5eb9193cba5339ee508dff867 2024-12-12T06:12:34,602 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for f5fe5dc5eb9193cba5339ee508dff867: Running coprocessor pre-open hook at 1733983954587Writing region info on filesystem at 1733983954587Initializing all the Stores at 1733983954589 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733983954589Cleaning up temporary data from old regions at 1733983954596 (+7 ms)Running coprocessor post-open hooks at 1733983954602 (+6 ms)Region opened successfully at 1733983954602 2024-12-12T06:12:34,603 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1733983953560.f5fe5dc5eb9193cba5339ee508dff867., pid=36, masterSystemTime=1733983954553 2024-12-12T06:12:34,606 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1733983953560.f5fe5dc5eb9193cba5339ee508dff867. 2024-12-12T06:12:34,606 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened test,rrr,1733983953560.f5fe5dc5eb9193cba5339ee508dff867. 2024-12-12T06:12:34,606 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open test,ggg,1733983953560.b717f65da42dacb67935407f094e8467. 2024-12-12T06:12:34,607 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => b717f65da42dacb67935407f094e8467, NAME => 'test,ggg,1733983953560.b717f65da42dacb67935407f094e8467.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-12-12T06:12:34,607 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=f5fe5dc5eb9193cba5339ee508dff867, regionState=OPEN, openSeqNum=2, regionLocation=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:34,607 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test b717f65da42dacb67935407f094e8467 2024-12-12T06:12:34,607 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated test,ggg,1733983953560.b717f65da42dacb67935407f094e8467.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-12T06:12:34,607 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for b717f65da42dacb67935407f094e8467 2024-12-12T06:12:34,607 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for b717f65da42dacb67935407f094e8467 2024-12-12T06:12:34,610 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5fe5dc5eb9193cba5339ee508dff867, server=ffbfd3107920,46199,1733983951048 because future has completed 2024-12-12T06:12:34,610 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-12T06:12:34,610 INFO [StoreOpener-b717f65da42dacb67935407f094e8467-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region b717f65da42dacb67935407f094e8467 2024-12-12T06:12:34,611 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39631, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-12T06:12:34,612 INFO [StoreOpener-b717f65da42dacb67935407f094e8467-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b717f65da42dacb67935407f094e8467 columnFamilyName family 2024-12-12T06:12:34,612 DEBUG [StoreOpener-b717f65da42dacb67935407f094e8467-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-12T06:12:34,613 INFO [StoreOpener-b717f65da42dacb67935407f094e8467-1 {}] regionserver.HStore(327): Store=b717f65da42dacb67935407f094e8467/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-12T06:12:34,613 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for b717f65da42dacb67935407f094e8467 2024-12-12T06:12:34,614 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/b717f65da42dacb67935407f094e8467 2024-12-12T06:12:34,615 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/b717f65da42dacb67935407f094e8467 2024-12-12T06:12:34,616 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for b717f65da42dacb67935407f094e8467 2024-12-12T06:12:34,616 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for b717f65da42dacb67935407f094e8467 2024-12-12T06:12:34,616 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=22 2024-12-12T06:12:34,616 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure f5fe5dc5eb9193cba5339ee508dff867, server=ffbfd3107920,46199,1733983951048 in 177 msec 2024-12-12T06:12:34,617 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open test,qqq,1733983953560.653572572648db72daea94dd65cfb7f8. 2024-12-12T06:12:34,617 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => 653572572648db72daea94dd65cfb7f8, NAME => 'test,qqq,1733983953560.653572572648db72daea94dd65cfb7f8.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-12-12T06:12:34,618 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 653572572648db72daea94dd65cfb7f8 2024-12-12T06:12:34,618 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated test,qqq,1733983953560.653572572648db72daea94dd65cfb7f8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-12T06:12:34,618 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for 653572572648db72daea94dd65cfb7f8 2024-12-12T06:12:34,618 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for 653572572648db72daea94dd65cfb7f8 2024-12-12T06:12:34,619 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for b717f65da42dacb67935407f094e8467 2024-12-12T06:12:34,619 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f5fe5dc5eb9193cba5339ee508dff867, ASSIGN in 392 msec 2024-12-12T06:12:34,620 INFO [StoreOpener-653572572648db72daea94dd65cfb7f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 653572572648db72daea94dd65cfb7f8 2024-12-12T06:12:34,621 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/b717f65da42dacb67935407f094e8467/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-12T06:12:34,622 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened b717f65da42dacb67935407f094e8467; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70777659, jitterRate=0.05466930568218231}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-12T06:12:34,622 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b717f65da42dacb67935407f094e8467 2024-12-12T06:12:34,622 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for b717f65da42dacb67935407f094e8467: Running coprocessor pre-open hook at 1733983954608Writing region info on filesystem at 1733983954608Initializing all the Stores at 1733983954610 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733983954610Cleaning up temporary data from old regions at 1733983954616 (+6 ms)Running coprocessor post-open hooks at 1733983954622 (+6 ms)Region opened successfully at 1733983954622 2024-12-12T06:12:34,623 INFO [StoreOpener-653572572648db72daea94dd65cfb7f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 653572572648db72daea94dd65cfb7f8 columnFamilyName family 2024-12-12T06:12:34,623 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ggg,1733983953560.b717f65da42dacb67935407f094e8467., pid=33, masterSystemTime=1733983954553 2024-12-12T06:12:34,623 DEBUG [StoreOpener-653572572648db72daea94dd65cfb7f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-12T06:12:34,624 INFO [StoreOpener-653572572648db72daea94dd65cfb7f8-1 {}] regionserver.HStore(327): Store=653572572648db72daea94dd65cfb7f8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-12T06:12:34,624 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for 653572572648db72daea94dd65cfb7f8 2024-12-12T06:12:34,626 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/653572572648db72daea94dd65cfb7f8 2024-12-12T06:12:34,626 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ggg,1733983953560.b717f65da42dacb67935407f094e8467. 2024-12-12T06:12:34,626 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened test,ggg,1733983953560.b717f65da42dacb67935407f094e8467. 2024-12-12T06:12:34,626 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/653572572648db72daea94dd65cfb7f8 2024-12-12T06:12:34,626 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open test,ooo,1733983953560.a4bf848f8297d89de7b203fcc6678eb5. 2024-12-12T06:12:34,627 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => a4bf848f8297d89de7b203fcc6678eb5, NAME => 'test,ooo,1733983953560.a4bf848f8297d89de7b203fcc6678eb5.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-12-12T06:12:34,627 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=b717f65da42dacb67935407f094e8467, regionState=OPEN, openSeqNum=2, regionLocation=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:34,627 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for 653572572648db72daea94dd65cfb7f8 2024-12-12T06:12:34,627 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test a4bf848f8297d89de7b203fcc6678eb5 2024-12-12T06:12:34,627 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for 653572572648db72daea94dd65cfb7f8 2024-12-12T06:12:34,627 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated test,ooo,1733983953560.a4bf848f8297d89de7b203fcc6678eb5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-12T06:12:34,627 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for a4bf848f8297d89de7b203fcc6678eb5 2024-12-12T06:12:34,628 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for a4bf848f8297d89de7b203fcc6678eb5 2024-12-12T06:12:34,630 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure b717f65da42dacb67935407f094e8467, server=ffbfd3107920,46199,1733983951048 because future has completed 2024-12-12T06:12:34,630 INFO [StoreOpener-a4bf848f8297d89de7b203fcc6678eb5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region a4bf848f8297d89de7b203fcc6678eb5 2024-12-12T06:12:34,630 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for 653572572648db72daea94dd65cfb7f8 2024-12-12T06:12:34,632 INFO [StoreOpener-a4bf848f8297d89de7b203fcc6678eb5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4bf848f8297d89de7b203fcc6678eb5 columnFamilyName family 2024-12-12T06:12:34,632 DEBUG [StoreOpener-a4bf848f8297d89de7b203fcc6678eb5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-12T06:12:34,633 INFO [StoreOpener-a4bf848f8297d89de7b203fcc6678eb5-1 {}] regionserver.HStore(327): Store=a4bf848f8297d89de7b203fcc6678eb5/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-12T06:12:34,633 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for a4bf848f8297d89de7b203fcc6678eb5 2024-12-12T06:12:34,634 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/653572572648db72daea94dd65cfb7f8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-12T06:12:34,635 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/a4bf848f8297d89de7b203fcc6678eb5 2024-12-12T06:12:34,635 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/a4bf848f8297d89de7b203fcc6678eb5 2024-12-12T06:12:34,635 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened 653572572648db72daea94dd65cfb7f8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69931823, jitterRate=0.042065367102622986}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-12T06:12:34,635 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 653572572648db72daea94dd65cfb7f8 2024-12-12T06:12:34,636 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for 653572572648db72daea94dd65cfb7f8: Running coprocessor pre-open hook at 1733983954618Writing region info on filesystem at 1733983954618Initializing all the Stores at 1733983954619 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733983954620 (+1 ms)Cleaning up temporary data from old regions at 1733983954627 (+7 ms)Running coprocessor post-open hooks at 1733983954635 (+8 ms)Region opened successfully at 1733983954636 (+1 ms) 2024-12-12T06:12:34,636 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for a4bf848f8297d89de7b203fcc6678eb5 2024-12-12T06:12:34,636 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for a4bf848f8297d89de7b203fcc6678eb5 2024-12-12T06:12:34,636 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=11 2024-12-12T06:12:34,636 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure b717f65da42dacb67935407f094e8467, server=ffbfd3107920,46199,1733983951048 in 213 msec 2024-12-12T06:12:34,637 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for test,qqq,1733983953560.653572572648db72daea94dd65cfb7f8., pid=40, masterSystemTime=1733983954609 2024-12-12T06:12:34,639 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for a4bf848f8297d89de7b203fcc6678eb5 2024-12-12T06:12:34,640 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for test,qqq,1733983953560.653572572648db72daea94dd65cfb7f8. 2024-12-12T06:12:34,640 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened test,qqq,1733983953560.653572572648db72daea94dd65cfb7f8. 2024-12-12T06:12:34,641 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(132): Open test,zzz,1733983953560.269eed2924060312949b51c10150f409. 2024-12-12T06:12:34,641 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7752): Opening region: {ENCODED => 269eed2924060312949b51c10150f409, NAME => 'test,zzz,1733983953560.269eed2924060312949b51c10150f409.', STARTKEY => 'zzz', ENDKEY => ''} 2024-12-12T06:12:34,641 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=b717f65da42dacb67935407f094e8467, ASSIGN in 413 msec 2024-12-12T06:12:34,641 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 269eed2924060312949b51c10150f409 2024-12-12T06:12:34,641 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(898): Instantiated test,zzz,1733983953560.269eed2924060312949b51c10150f409.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-12T06:12:34,641 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7794): checking encryption for 269eed2924060312949b51c10150f409 2024-12-12T06:12:34,642 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7797): checking classloading for 269eed2924060312949b51c10150f409 2024-12-12T06:12:34,643 INFO [StoreOpener-269eed2924060312949b51c10150f409-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 269eed2924060312949b51c10150f409 2024-12-12T06:12:34,644 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=653572572648db72daea94dd65cfb7f8, regionState=OPEN, openSeqNum=2, regionLocation=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:34,644 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/a4bf848f8297d89de7b203fcc6678eb5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-12T06:12:34,646 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened a4bf848f8297d89de7b203fcc6678eb5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63575976, jitterRate=-0.05264413356781006}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-12T06:12:34,646 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a4bf848f8297d89de7b203fcc6678eb5 2024-12-12T06:12:34,646 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for a4bf848f8297d89de7b203fcc6678eb5: Running coprocessor pre-open hook at 1733983954628Writing region info on filesystem at 1733983954628Initializing all the Stores at 1733983954629 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733983954629Cleaning up temporary data from old regions at 1733983954636 (+7 ms)Running coprocessor post-open hooks at 1733983954646 (+10 ms)Region opened successfully at 1733983954646 2024-12-12T06:12:34,647 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ooo,1733983953560.a4bf848f8297d89de7b203fcc6678eb5., pid=35, masterSystemTime=1733983954553 2024-12-12T06:12:34,647 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 653572572648db72daea94dd65cfb7f8, server=ffbfd3107920,42275,1733983951224 because future has completed 2024-12-12T06:12:34,647 INFO [StoreOpener-269eed2924060312949b51c10150f409-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 269eed2924060312949b51c10150f409 columnFamilyName family 2024-12-12T06:12:34,648 DEBUG [StoreOpener-269eed2924060312949b51c10150f409-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-12T06:12:34,649 INFO [StoreOpener-269eed2924060312949b51c10150f409-1 {}] regionserver.HStore(327): Store=269eed2924060312949b51c10150f409/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-12T06:12:34,650 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ooo,1733983953560.a4bf848f8297d89de7b203fcc6678eb5. 2024-12-12T06:12:34,650 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1038): replaying wal for 269eed2924060312949b51c10150f409 2024-12-12T06:12:34,650 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened test,ooo,1733983953560.a4bf848f8297d89de7b203fcc6678eb5. 2024-12-12T06:12:34,650 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open test,ddd,1733983953560.c98bb859cc9e047b211ddb4c5b967ce5. 2024-12-12T06:12:34,650 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => c98bb859cc9e047b211ddb4c5b967ce5, NAME => 'test,ddd,1733983953560.c98bb859cc9e047b211ddb4c5b967ce5.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-12-12T06:12:34,651 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c98bb859cc9e047b211ddb4c5b967ce5 2024-12-12T06:12:34,651 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated test,ddd,1733983953560.c98bb859cc9e047b211ddb4c5b967ce5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-12T06:12:34,651 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open test,yyy,1733983953560.ae428798a6433b4d48a205ab3d14ba42. 2024-12-12T06:12:34,651 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for c98bb859cc9e047b211ddb4c5b967ce5 2024-12-12T06:12:34,651 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for c98bb859cc9e047b211ddb4c5b967ce5 2024-12-12T06:12:34,651 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/269eed2924060312949b51c10150f409 2024-12-12T06:12:34,652 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => ae428798a6433b4d48a205ab3d14ba42, NAME => 'test,yyy,1733983953560.ae428798a6433b4d48a205ab3d14ba42.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-12-12T06:12:34,652 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/269eed2924060312949b51c10150f409 2024-12-12T06:12:34,653 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1048): stopping wal replay for 269eed2924060312949b51c10150f409 2024-12-12T06:12:34,652 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ae428798a6433b4d48a205ab3d14ba42 2024-12-12T06:12:34,653 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1060): Cleaning up temporary data for 269eed2924060312949b51c10150f409 2024-12-12T06:12:34,653 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated test,yyy,1733983953560.ae428798a6433b4d48a205ab3d14ba42.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-12T06:12:34,653 INFO [StoreOpener-c98bb859cc9e047b211ddb4c5b967ce5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c98bb859cc9e047b211ddb4c5b967ce5 2024-12-12T06:12:34,653 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for ae428798a6433b4d48a205ab3d14ba42 2024-12-12T06:12:34,653 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for ae428798a6433b4d48a205ab3d14ba42 2024-12-12T06:12:34,653 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=21 2024-12-12T06:12:34,653 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=21, state=SUCCESS, hasLock=false; OpenRegionProcedure 653572572648db72daea94dd65cfb7f8, server=ffbfd3107920,42275,1733983951224 in 188 msec 2024-12-12T06:12:34,655 INFO [StoreOpener-c98bb859cc9e047b211ddb4c5b967ce5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c98bb859cc9e047b211ddb4c5b967ce5 columnFamilyName family 2024-12-12T06:12:34,655 DEBUG [StoreOpener-c98bb859cc9e047b211ddb4c5b967ce5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-12T06:12:34,656 INFO [StoreOpener-c98bb859cc9e047b211ddb4c5b967ce5-1 {}] regionserver.HStore(327): Store=c98bb859cc9e047b211ddb4c5b967ce5/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-12T06:12:34,656 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1093): writing seq id for 269eed2924060312949b51c10150f409 2024-12-12T06:12:34,656 INFO [StoreOpener-ae428798a6433b4d48a205ab3d14ba42-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ae428798a6433b4d48a205ab3d14ba42 2024-12-12T06:12:34,656 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for c98bb859cc9e047b211ddb4c5b967ce5 2024-12-12T06:12:34,656 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=653572572648db72daea94dd65cfb7f8, ASSIGN in 429 msec 2024-12-12T06:12:34,658 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/c98bb859cc9e047b211ddb4c5b967ce5 2024-12-12T06:12:34,659 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=a4bf848f8297d89de7b203fcc6678eb5, regionState=OPEN, openSeqNum=2, regionLocation=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:34,659 INFO [StoreOpener-ae428798a6433b4d48a205ab3d14ba42-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ae428798a6433b4d48a205ab3d14ba42 columnFamilyName family 2024-12-12T06:12:34,659 DEBUG [StoreOpener-ae428798a6433b4d48a205ab3d14ba42-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-12T06:12:34,659 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/269eed2924060312949b51c10150f409/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-12T06:12:34,660 INFO [StoreOpener-ae428798a6433b4d48a205ab3d14ba42-1 {}] regionserver.HStore(327): Store=ae428798a6433b4d48a205ab3d14ba42/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-12T06:12:34,660 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for ae428798a6433b4d48a205ab3d14ba42 2024-12-12T06:12:34,660 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1114): Opened 269eed2924060312949b51c10150f409; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60858828, jitterRate=-0.09313279390335083}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-12T06:12:34,660 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 269eed2924060312949b51c10150f409 2024-12-12T06:12:34,661 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1006): Region open journal for 269eed2924060312949b51c10150f409: Running coprocessor pre-open hook at 1733983954642Writing region info on filesystem at 1733983954642Initializing all the Stores at 1733983954643 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733983954643Cleaning up temporary data from old regions at 1733983954653 (+10 ms)Running coprocessor post-open hooks at 1733983954660 (+7 ms)Region opened successfully at 1733983954660 2024-12-12T06:12:34,661 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/c98bb859cc9e047b211ddb4c5b967ce5 2024-12-12T06:12:34,661 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/ae428798a6433b4d48a205ab3d14ba42 2024-12-12T06:12:34,662 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for c98bb859cc9e047b211ddb4c5b967ce5 2024-12-12T06:12:34,662 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for c98bb859cc9e047b211ddb4c5b967ce5 2024-12-12T06:12:34,662 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2236): Post open deploy tasks for test,zzz,1733983953560.269eed2924060312949b51c10150f409., pid=42, masterSystemTime=1733983954609 2024-12-12T06:12:34,662 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure a4bf848f8297d89de7b203fcc6678eb5, server=ffbfd3107920,46199,1733983951048 because future has completed 2024-12-12T06:12:34,662 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/ae428798a6433b4d48a205ab3d14ba42 2024-12-12T06:12:34,664 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for ae428798a6433b4d48a205ab3d14ba42 2024-12-12T06:12:34,664 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for ae428798a6433b4d48a205ab3d14ba42 2024-12-12T06:12:34,665 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for c98bb859cc9e047b211ddb4c5b967ce5 2024-12-12T06:12:34,665 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2266): Finished post open deploy task for test,zzz,1733983953560.269eed2924060312949b51c10150f409. 2024-12-12T06:12:34,665 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(153): Opened test,zzz,1733983953560.269eed2924060312949b51c10150f409. 2024-12-12T06:12:34,665 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open test,ttt,1733983953560.5162d3c0b3255050cbeed27e4456e5c2. 2024-12-12T06:12:34,666 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => 5162d3c0b3255050cbeed27e4456e5c2, NAME => 'test,ttt,1733983953560.5162d3c0b3255050cbeed27e4456e5c2.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-12-12T06:12:34,666 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 5162d3c0b3255050cbeed27e4456e5c2 2024-12-12T06:12:34,666 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated test,ttt,1733983953560.5162d3c0b3255050cbeed27e4456e5c2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-12T06:12:34,666 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for 5162d3c0b3255050cbeed27e4456e5c2 2024-12-12T06:12:34,666 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for 5162d3c0b3255050cbeed27e4456e5c2 2024-12-12T06:12:34,668 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=19 2024-12-12T06:12:34,668 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/c98bb859cc9e047b211ddb4c5b967ce5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-12T06:12:34,668 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=19, state=SUCCESS, hasLock=false; OpenRegionProcedure a4bf848f8297d89de7b203fcc6678eb5, server=ffbfd3107920,46199,1733983951048 in 234 msec 2024-12-12T06:12:34,669 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened c98bb859cc9e047b211ddb4c5b967ce5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75445098, jitterRate=0.12421956658363342}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-12T06:12:34,669 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c98bb859cc9e047b211ddb4c5b967ce5 2024-12-12T06:12:34,669 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for ae428798a6433b4d48a205ab3d14ba42 2024-12-12T06:12:34,669 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for c98bb859cc9e047b211ddb4c5b967ce5: Running coprocessor pre-open hook at 1733983954651Writing region info on filesystem at 1733983954651Initializing all the Stores at 1733983954652 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733983954652Cleaning up temporary data from old regions at 1733983954662 (+10 ms)Running coprocessor post-open hooks at 1733983954669 (+7 ms)Region opened successfully at 1733983954669 2024-12-12T06:12:34,670 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ddd,1733983953560.c98bb859cc9e047b211ddb4c5b967ce5., pid=38, masterSystemTime=1733983954553 2024-12-12T06:12:34,671 INFO [StoreOpener-5162d3c0b3255050cbeed27e4456e5c2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 5162d3c0b3255050cbeed27e4456e5c2 2024-12-12T06:12:34,672 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a4bf848f8297d89de7b203fcc6678eb5, ASSIGN in 445 msec 2024-12-12T06:12:34,673 INFO [StoreOpener-5162d3c0b3255050cbeed27e4456e5c2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5162d3c0b3255050cbeed27e4456e5c2 columnFamilyName family 2024-12-12T06:12:34,674 DEBUG [StoreOpener-5162d3c0b3255050cbeed27e4456e5c2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-12T06:12:34,674 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=269eed2924060312949b51c10150f409, regionState=OPEN, openSeqNum=2, regionLocation=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:34,674 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ddd,1733983953560.c98bb859cc9e047b211ddb4c5b967ce5. 2024-12-12T06:12:34,674 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened test,ddd,1733983953560.c98bb859cc9e047b211ddb4c5b967ce5. 2024-12-12T06:12:34,674 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open test,ccc,1733983953560.2eeaafb9a1964f4c6899fa5dde05a463. 2024-12-12T06:12:34,674 INFO [StoreOpener-5162d3c0b3255050cbeed27e4456e5c2-1 {}] regionserver.HStore(327): Store=5162d3c0b3255050cbeed27e4456e5c2/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-12T06:12:34,675 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => 2eeaafb9a1964f4c6899fa5dde05a463, NAME => 'test,ccc,1733983953560.2eeaafb9a1964f4c6899fa5dde05a463.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-12-12T06:12:34,675 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for 5162d3c0b3255050cbeed27e4456e5c2 2024-12-12T06:12:34,675 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 2eeaafb9a1964f4c6899fa5dde05a463 2024-12-12T06:12:34,675 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated test,ccc,1733983953560.2eeaafb9a1964f4c6899fa5dde05a463.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-12T06:12:34,675 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for 2eeaafb9a1964f4c6899fa5dde05a463 2024-12-12T06:12:34,675 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for 2eeaafb9a1964f4c6899fa5dde05a463 2024-12-12T06:12:34,676 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/5162d3c0b3255050cbeed27e4456e5c2 2024-12-12T06:12:34,677 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=c98bb859cc9e047b211ddb4c5b967ce5, regionState=OPEN, openSeqNum=2, regionLocation=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:34,677 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/5162d3c0b3255050cbeed27e4456e5c2 2024-12-12T06:12:34,678 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 269eed2924060312949b51c10150f409, server=ffbfd3107920,42275,1733983951224 because future has completed 2024-12-12T06:12:34,678 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for 5162d3c0b3255050cbeed27e4456e5c2 2024-12-12T06:12:34,678 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for 5162d3c0b3255050cbeed27e4456e5c2 2024-12-12T06:12:34,678 INFO [StoreOpener-2eeaafb9a1964f4c6899fa5dde05a463-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 2eeaafb9a1964f4c6899fa5dde05a463 2024-12-12T06:12:34,679 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/ae428798a6433b4d48a205ab3d14ba42/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-12T06:12:34,680 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened ae428798a6433b4d48a205ab3d14ba42; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72016563, jitterRate=0.07313041388988495}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-12T06:12:34,681 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ae428798a6433b4d48a205ab3d14ba42 2024-12-12T06:12:34,681 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for ae428798a6433b4d48a205ab3d14ba42: Running coprocessor pre-open hook at 1733983954653Writing region info on filesystem at 1733983954653Initializing all the Stores at 1733983954655 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733983954655Cleaning up temporary data from old regions at 1733983954664 (+9 ms)Running coprocessor post-open hooks at 1733983954681 (+17 ms)Region opened successfully at 1733983954681 2024-12-12T06:12:34,681 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure c98bb859cc9e047b211ddb4c5b967ce5, server=ffbfd3107920,46199,1733983951048 because future has completed 2024-12-12T06:12:34,681 INFO [StoreOpener-2eeaafb9a1964f4c6899fa5dde05a463-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2eeaafb9a1964f4c6899fa5dde05a463 columnFamilyName family 2024-12-12T06:12:34,681 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for 5162d3c0b3255050cbeed27e4456e5c2 2024-12-12T06:12:34,681 DEBUG [StoreOpener-2eeaafb9a1964f4c6899fa5dde05a463-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-12T06:12:34,682 INFO [StoreOpener-2eeaafb9a1964f4c6899fa5dde05a463-1 {}] regionserver.HStore(327): Store=2eeaafb9a1964f4c6899fa5dde05a463/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-12T06:12:34,683 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for 2eeaafb9a1964f4c6899fa5dde05a463 2024-12-12T06:12:34,683 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1733983953560.ae428798a6433b4d48a205ab3d14ba42., pid=45, masterSystemTime=1733983954640 2024-12-12T06:12:34,685 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/2eeaafb9a1964f4c6899fa5dde05a463 2024-12-12T06:12:34,686 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/5162d3c0b3255050cbeed27e4456e5c2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-12T06:12:34,686 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1733983953560.ae428798a6433b4d48a205ab3d14ba42. 2024-12-12T06:12:34,686 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/2eeaafb9a1964f4c6899fa5dde05a463 2024-12-12T06:12:34,687 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened test,yyy,1733983953560.ae428798a6433b4d48a205ab3d14ba42. 2024-12-12T06:12:34,687 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open test,vvv,1733983953560.60c948f12e02b6ab0066426b6aceea80. 2024-12-12T06:12:34,687 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened 5162d3c0b3255050cbeed27e4456e5c2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69437521, jitterRate=0.0346996933221817}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-12T06:12:34,687 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=30 2024-12-12T06:12:34,687 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5162d3c0b3255050cbeed27e4456e5c2 2024-12-12T06:12:34,687 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => 60c948f12e02b6ab0066426b6aceea80, NAME => 'test,vvv,1733983953560.60c948f12e02b6ab0066426b6aceea80.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-12-12T06:12:34,687 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure 269eed2924060312949b51c10150f409, server=ffbfd3107920,42275,1733983951224 in 212 msec 2024-12-12T06:12:34,687 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for 5162d3c0b3255050cbeed27e4456e5c2: Running coprocessor pre-open hook at 1733983954666Writing region info on filesystem at 1733983954666Initializing all the Stores at 1733983954668 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733983954668Cleaning up temporary data from old regions at 1733983954678 (+10 ms)Running coprocessor post-open hooks at 1733983954687 (+9 ms)Region opened successfully at 1733983954687 2024-12-12T06:12:34,687 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for 2eeaafb9a1964f4c6899fa5dde05a463 2024-12-12T06:12:34,687 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for 2eeaafb9a1964f4c6899fa5dde05a463 2024-12-12T06:12:34,687 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 60c948f12e02b6ab0066426b6aceea80 2024-12-12T06:12:34,687 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated test,vvv,1733983953560.60c948f12e02b6ab0066426b6aceea80.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-12T06:12:34,688 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for 60c948f12e02b6ab0066426b6aceea80 2024-12-12T06:12:34,688 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for 60c948f12e02b6ab0066426b6aceea80 2024-12-12T06:12:34,689 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1733983953560.5162d3c0b3255050cbeed27e4456e5c2., pid=39, masterSystemTime=1733983954609 2024-12-12T06:12:34,690 INFO [StoreOpener-60c948f12e02b6ab0066426b6aceea80-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 60c948f12e02b6ab0066426b6aceea80 2024-12-12T06:12:34,690 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for 2eeaafb9a1964f4c6899fa5dde05a463 2024-12-12T06:12:34,690 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=269eed2924060312949b51c10150f409, ASSIGN in 463 msec 2024-12-12T06:12:34,692 INFO [StoreOpener-60c948f12e02b6ab0066426b6aceea80-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 60c948f12e02b6ab0066426b6aceea80 columnFamilyName family 2024-12-12T06:12:34,692 DEBUG [StoreOpener-60c948f12e02b6ab0066426b6aceea80-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-12T06:12:34,692 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1733983953560.5162d3c0b3255050cbeed27e4456e5c2. 2024-12-12T06:12:34,692 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened test,ttt,1733983953560.5162d3c0b3255050cbeed27e4456e5c2. 2024-12-12T06:12:34,693 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(132): Open test,hhh,1733983953560.f2b6a6b11f14670aca9dccceb20d9630. 2024-12-12T06:12:34,693 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7752): Opening region: {ENCODED => f2b6a6b11f14670aca9dccceb20d9630, NAME => 'test,hhh,1733983953560.f2b6a6b11f14670aca9dccceb20d9630.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-12-12T06:12:34,693 INFO [StoreOpener-60c948f12e02b6ab0066426b6aceea80-1 {}] regionserver.HStore(327): Store=60c948f12e02b6ab0066426b6aceea80/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-12T06:12:34,693 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=ae428798a6433b4d48a205ab3d14ba42, regionState=OPEN, openSeqNum=2, regionLocation=ffbfd3107920,33455,1733983951151 2024-12-12T06:12:34,693 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for 60c948f12e02b6ab0066426b6aceea80 2024-12-12T06:12:34,693 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f2b6a6b11f14670aca9dccceb20d9630 2024-12-12T06:12:34,693 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(898): Instantiated test,hhh,1733983953560.f2b6a6b11f14670aca9dccceb20d9630.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-12T06:12:34,694 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7794): checking encryption for f2b6a6b11f14670aca9dccceb20d9630 2024-12-12T06:12:34,694 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/2eeaafb9a1964f4c6899fa5dde05a463/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-12T06:12:34,694 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7797): checking classloading for f2b6a6b11f14670aca9dccceb20d9630 2024-12-12T06:12:34,694 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/60c948f12e02b6ab0066426b6aceea80 2024-12-12T06:12:34,694 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened 2eeaafb9a1964f4c6899fa5dde05a463; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72059843, jitterRate=0.07377533614635468}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-12T06:12:34,695 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2eeaafb9a1964f4c6899fa5dde05a463 2024-12-12T06:12:34,695 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for 2eeaafb9a1964f4c6899fa5dde05a463: Running coprocessor pre-open hook at 1733983954675Writing region info on filesystem at 1733983954675Initializing all the Stores at 1733983954677 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733983954677Cleaning up temporary data from old regions at 1733983954687 (+10 ms)Running coprocessor post-open hooks at 1733983954695 (+8 ms)Region opened successfully at 1733983954695 2024-12-12T06:12:34,695 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/60c948f12e02b6ab0066426b6aceea80 2024-12-12T06:12:34,696 INFO [StoreOpener-f2b6a6b11f14670aca9dccceb20d9630-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f2b6a6b11f14670aca9dccceb20d9630 2024-12-12T06:12:34,696 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for 60c948f12e02b6ab0066426b6aceea80 2024-12-12T06:12:34,696 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for 60c948f12e02b6ab0066426b6aceea80 2024-12-12T06:12:34,697 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1733983953560.2eeaafb9a1964f4c6899fa5dde05a463., pid=43, masterSystemTime=1733983954553 2024-12-12T06:12:34,697 INFO [StoreOpener-f2b6a6b11f14670aca9dccceb20d9630-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f2b6a6b11f14670aca9dccceb20d9630 columnFamilyName family 2024-12-12T06:12:34,698 DEBUG [StoreOpener-f2b6a6b11f14670aca9dccceb20d9630-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-12T06:12:34,698 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure ae428798a6433b4d48a205ab3d14ba42, server=ffbfd3107920,33455,1733983951151 because future has completed 2024-12-12T06:12:34,699 INFO [StoreOpener-f2b6a6b11f14670aca9dccceb20d9630-1 {}] regionserver.HStore(327): Store=f2b6a6b11f14670aca9dccceb20d9630/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-12T06:12:34,699 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=5162d3c0b3255050cbeed27e4456e5c2, regionState=OPEN, openSeqNum=2, regionLocation=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:34,699 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1038): replaying wal for f2b6a6b11f14670aca9dccceb20d9630 2024-12-12T06:12:34,700 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ccc,1733983953560.2eeaafb9a1964f4c6899fa5dde05a463. 2024-12-12T06:12:34,700 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened test,ccc,1733983953560.2eeaafb9a1964f4c6899fa5dde05a463. 2024-12-12T06:12:34,700 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open test,kkk,1733983953560.782c52424de7d71cb43b8fbfe8e89f7f. 2024-12-12T06:12:34,701 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 782c52424de7d71cb43b8fbfe8e89f7f, NAME => 'test,kkk,1733983953560.782c52424de7d71cb43b8fbfe8e89f7f.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-12-12T06:12:34,701 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/f2b6a6b11f14670aca9dccceb20d9630 2024-12-12T06:12:34,701 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for 60c948f12e02b6ab0066426b6aceea80 2024-12-12T06:12:34,702 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 782c52424de7d71cb43b8fbfe8e89f7f 2024-12-12T06:12:34,702 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated test,kkk,1733983953560.782c52424de7d71cb43b8fbfe8e89f7f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-12T06:12:34,702 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 782c52424de7d71cb43b8fbfe8e89f7f 2024-12-12T06:12:34,702 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 782c52424de7d71cb43b8fbfe8e89f7f 2024-12-12T06:12:34,702 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5162d3c0b3255050cbeed27e4456e5c2, server=ffbfd3107920,42275,1733983951224 because future has completed 2024-12-12T06:12:34,703 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/f2b6a6b11f14670aca9dccceb20d9630 2024-12-12T06:12:34,703 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=2eeaafb9a1964f4c6899fa5dde05a463, regionState=OPEN, openSeqNum=2, regionLocation=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:34,703 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1048): stopping wal replay for f2b6a6b11f14670aca9dccceb20d9630 2024-12-12T06:12:34,703 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1060): Cleaning up temporary data for f2b6a6b11f14670aca9dccceb20d9630 2024-12-12T06:12:34,704 INFO [StoreOpener-782c52424de7d71cb43b8fbfe8e89f7f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 782c52424de7d71cb43b8fbfe8e89f7f 2024-12-12T06:12:34,706 INFO [StoreOpener-782c52424de7d71cb43b8fbfe8e89f7f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 782c52424de7d71cb43b8fbfe8e89f7f columnFamilyName family 2024-12-12T06:12:34,707 DEBUG [StoreOpener-782c52424de7d71cb43b8fbfe8e89f7f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-12T06:12:34,708 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/60c948f12e02b6ab0066426b6aceea80/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-12T06:12:34,708 INFO [StoreOpener-782c52424de7d71cb43b8fbfe8e89f7f-1 {}] regionserver.HStore(327): Store=782c52424de7d71cb43b8fbfe8e89f7f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-12T06:12:34,708 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 782c52424de7d71cb43b8fbfe8e89f7f 2024-12-12T06:12:34,708 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2eeaafb9a1964f4c6899fa5dde05a463, server=ffbfd3107920,46199,1733983951048 because future has completed 2024-12-12T06:12:34,709 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): writing seq id for f2b6a6b11f14670aca9dccceb20d9630 2024-12-12T06:12:34,709 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened 60c948f12e02b6ab0066426b6aceea80; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60111457, jitterRate=-0.10426948964595795}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-12T06:12:34,709 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 60c948f12e02b6ab0066426b6aceea80 2024-12-12T06:12:34,709 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for 60c948f12e02b6ab0066426b6aceea80: Running coprocessor pre-open hook at 1733983954688Writing region info on filesystem at 1733983954688Initializing all the Stores at 1733983954689 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733983954689Cleaning up temporary data from old regions at 1733983954696 (+7 ms)Running coprocessor post-open hooks at 1733983954709 (+13 ms)Region opened successfully at 1733983954709 2024-12-12T06:12:34,709 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/782c52424de7d71cb43b8fbfe8e89f7f 2024-12-12T06:12:34,711 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1733983953560.60c948f12e02b6ab0066426b6aceea80., pid=46, masterSystemTime=1733983954640 2024-12-12T06:12:34,711 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/782c52424de7d71cb43b8fbfe8e89f7f 2024-12-12T06:12:34,712 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 782c52424de7d71cb43b8fbfe8e89f7f 2024-12-12T06:12:34,712 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 782c52424de7d71cb43b8fbfe8e89f7f 2024-12-12T06:12:34,713 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/f2b6a6b11f14670aca9dccceb20d9630/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-12T06:12:34,714 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1733983953560.60c948f12e02b6ab0066426b6aceea80. 2024-12-12T06:12:34,714 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1114): Opened f2b6a6b11f14670aca9dccceb20d9630; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73267177, jitterRate=0.09176601469516754}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-12T06:12:34,715 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened test,vvv,1733983953560.60c948f12e02b6ab0066426b6aceea80. 2024-12-12T06:12:34,715 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f2b6a6b11f14670aca9dccceb20d9630 2024-12-12T06:12:34,715 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open test,lll,1733983953560.5d34bdb8a1414fb25a475fb97fc440b4. 2024-12-12T06:12:34,715 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1006): Region open journal for f2b6a6b11f14670aca9dccceb20d9630: Running coprocessor pre-open hook at 1733983954694Writing region info on filesystem at 1733983954694Initializing all the Stores at 1733983954695 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733983954695Cleaning up temporary data from old regions at 1733983954703 (+8 ms)Running coprocessor post-open hooks at 1733983954715 (+12 ms)Region opened successfully at 1733983954715 2024-12-12T06:12:34,715 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => 5d34bdb8a1414fb25a475fb97fc440b4, NAME => 'test,lll,1733983953560.5d34bdb8a1414fb25a475fb97fc440b4.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-12-12T06:12:34,715 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=24 2024-12-12T06:12:34,715 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 5d34bdb8a1414fb25a475fb97fc440b4 2024-12-12T06:12:34,715 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure 5162d3c0b3255050cbeed27e4456e5c2, server=ffbfd3107920,42275,1733983951224 in 252 msec 2024-12-12T06:12:34,715 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated test,lll,1733983953560.5d34bdb8a1414fb25a475fb97fc440b4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-12T06:12:34,716 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for 5d34bdb8a1414fb25a475fb97fc440b4 2024-12-12T06:12:34,716 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for 5d34bdb8a1414fb25a475fb97fc440b4 2024-12-12T06:12:34,717 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 782c52424de7d71cb43b8fbfe8e89f7f 2024-12-12T06:12:34,717 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1733983953560.f2b6a6b11f14670aca9dccceb20d9630., pid=49, masterSystemTime=1733983954609 2024-12-12T06:12:34,718 INFO [StoreOpener-5d34bdb8a1414fb25a475fb97fc440b4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 5d34bdb8a1414fb25a475fb97fc440b4 2024-12-12T06:12:34,719 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=5162d3c0b3255050cbeed27e4456e5c2, ASSIGN in 492 msec 2024-12-12T06:12:34,721 INFO [StoreOpener-5d34bdb8a1414fb25a475fb97fc440b4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5d34bdb8a1414fb25a475fb97fc440b4 columnFamilyName family 2024-12-12T06:12:34,721 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/782c52424de7d71cb43b8fbfe8e89f7f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-12T06:12:34,721 DEBUG [StoreOpener-5d34bdb8a1414fb25a475fb97fc440b4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-12T06:12:34,722 INFO [StoreOpener-5d34bdb8a1414fb25a475fb97fc440b4-1 {}] regionserver.HStore(327): Store=5d34bdb8a1414fb25a475fb97fc440b4/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-12T06:12:34,722 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=60c948f12e02b6ab0066426b6aceea80, regionState=OPEN, openSeqNum=2, regionLocation=ffbfd3107920,33455,1733983951151 2024-12-12T06:12:34,722 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for 5d34bdb8a1414fb25a475fb97fc440b4 2024-12-12T06:12:34,722 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 782c52424de7d71cb43b8fbfe8e89f7f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74905494, jitterRate=0.11617884039878845}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-12T06:12:34,722 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2266): Finished post open deploy task for test,hhh,1733983953560.f2b6a6b11f14670aca9dccceb20d9630. 2024-12-12T06:12:34,722 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 782c52424de7d71cb43b8fbfe8e89f7f 2024-12-12T06:12:34,722 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(153): Opened test,hhh,1733983953560.f2b6a6b11f14670aca9dccceb20d9630. 2024-12-12T06:12:34,722 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 782c52424de7d71cb43b8fbfe8e89f7f: Running coprocessor pre-open hook at 1733983954702Writing region info on filesystem at 1733983954702Initializing all the Stores at 1733983954704 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733983954704Cleaning up temporary data from old regions at 1733983954712 (+8 ms)Running coprocessor post-open hooks at 1733983954722 (+10 ms)Region opened successfully at 1733983954722 2024-12-12T06:12:34,722 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open test,bbb,1733983953560.16838c5df0ab4e3938ad84e9a080def9. 2024-12-12T06:12:34,723 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => 16838c5df0ab4e3938ad84e9a080def9, NAME => 'test,bbb,1733983953560.16838c5df0ab4e3938ad84e9a080def9.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-12-12T06:12:34,723 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 16838c5df0ab4e3938ad84e9a080def9 2024-12-12T06:12:34,723 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated test,bbb,1733983953560.16838c5df0ab4e3938ad84e9a080def9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-12T06:12:34,723 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for 16838c5df0ab4e3938ad84e9a080def9 2024-12-12T06:12:34,723 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for 16838c5df0ab4e3938ad84e9a080def9 2024-12-12T06:12:34,724 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for test,kkk,1733983953560.782c52424de7d71cb43b8fbfe8e89f7f., pid=32, masterSystemTime=1733983954553 2024-12-12T06:12:34,724 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/5d34bdb8a1414fb25a475fb97fc440b4 2024-12-12T06:12:34,725 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/5d34bdb8a1414fb25a475fb97fc440b4 2024-12-12T06:12:34,725 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 60c948f12e02b6ab0066426b6aceea80, server=ffbfd3107920,33455,1733983951151 because future has completed 2024-12-12T06:12:34,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45209 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-12T06:12:34,727 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for 5d34bdb8a1414fb25a475fb97fc440b4 2024-12-12T06:12:34,727 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for 5d34bdb8a1414fb25a475fb97fc440b4 2024-12-12T06:12:34,727 INFO [StoreOpener-16838c5df0ab4e3938ad84e9a080def9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 16838c5df0ab4e3938ad84e9a080def9 2024-12-12T06:12:34,728 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=f2b6a6b11f14670aca9dccceb20d9630, regionState=OPEN, openSeqNum=2, regionLocation=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:34,728 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for test,kkk,1733983953560.782c52424de7d71cb43b8fbfe8e89f7f. 2024-12-12T06:12:34,728 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened test,kkk,1733983953560.782c52424de7d71cb43b8fbfe8e89f7f. 2024-12-12T06:12:34,728 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open test,uuu,1733983953560.d3ce5f64ef1889a3856db9d8c0f63a72. 2024-12-12T06:12:34,728 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => d3ce5f64ef1889a3856db9d8c0f63a72, NAME => 'test,uuu,1733983953560.d3ce5f64ef1889a3856db9d8c0f63a72.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-12-12T06:12:34,729 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d3ce5f64ef1889a3856db9d8c0f63a72 2024-12-12T06:12:34,729 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated test,uuu,1733983953560.d3ce5f64ef1889a3856db9d8c0f63a72.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-12T06:12:34,729 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for d3ce5f64ef1889a3856db9d8c0f63a72 2024-12-12T06:12:34,729 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for d3ce5f64ef1889a3856db9d8c0f63a72 2024-12-12T06:12:34,732 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure f2b6a6b11f14670aca9dccceb20d9630, server=ffbfd3107920,42275,1733983951224 because future has completed 2024-12-12T06:12:34,732 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for 5d34bdb8a1414fb25a475fb97fc440b4 2024-12-12T06:12:34,732 INFO [StoreOpener-16838c5df0ab4e3938ad84e9a080def9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 16838c5df0ab4e3938ad84e9a080def9 columnFamilyName family 2024-12-12T06:12:34,733 DEBUG [StoreOpener-16838c5df0ab4e3938ad84e9a080def9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-12T06:12:34,733 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=782c52424de7d71cb43b8fbfe8e89f7f, regionState=OPEN, openSeqNum=2, regionLocation=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:34,733 INFO [StoreOpener-16838c5df0ab4e3938ad84e9a080def9-1 {}] regionserver.HStore(327): Store=16838c5df0ab4e3938ad84e9a080def9/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-12T06:12:34,733 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for 16838c5df0ab4e3938ad84e9a080def9 2024-12-12T06:12:34,735 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/16838c5df0ab4e3938ad84e9a080def9 2024-12-12T06:12:34,735 INFO [StoreOpener-d3ce5f64ef1889a3856db9d8c0f63a72-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d3ce5f64ef1889a3856db9d8c0f63a72 2024-12-12T06:12:34,736 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/16838c5df0ab4e3938ad84e9a080def9 2024-12-12T06:12:34,737 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/5d34bdb8a1414fb25a475fb97fc440b4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-12T06:12:34,737 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 782c52424de7d71cb43b8fbfe8e89f7f, server=ffbfd3107920,46199,1733983951048 because future has completed 2024-12-12T06:12:34,737 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for 16838c5df0ab4e3938ad84e9a080def9 2024-12-12T06:12:34,737 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for 16838c5df0ab4e3938ad84e9a080def9 2024-12-12T06:12:34,739 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened 5d34bdb8a1414fb25a475fb97fc440b4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65754705, jitterRate=-0.02017854154109955}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-12T06:12:34,739 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5d34bdb8a1414fb25a475fb97fc440b4 2024-12-12T06:12:34,739 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for 5d34bdb8a1414fb25a475fb97fc440b4: Running coprocessor pre-open hook at 1733983954716Writing region info on filesystem at 1733983954716Initializing all the Stores at 1733983954717 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733983954717Cleaning up temporary data from old regions at 1733983954727 (+10 ms)Running coprocessor post-open hooks at 1733983954739 (+12 ms)Region opened successfully at 1733983954739 2024-12-12T06:12:34,740 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45209 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=ffbfd3107920,46199,1733983951048, table=test, region=2eeaafb9a1964f4c6899fa5dde05a463. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-12T06:12:34,740 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45209 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=ffbfd3107920,46199,1733983951048, table=test, region=c98bb859cc9e047b211ddb4c5b967ce5. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-12T06:12:34,740 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45209 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=ffbfd3107920,46199,1733983951048, table=test, region=782c52424de7d71cb43b8fbfe8e89f7f. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-12T06:12:34,741 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1733983953560.5d34bdb8a1414fb25a475fb97fc440b4., pid=53, masterSystemTime=1733983954640 2024-12-12T06:12:34,742 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for 16838c5df0ab4e3938ad84e9a080def9 2024-12-12T06:12:34,742 INFO [StoreOpener-d3ce5f64ef1889a3856db9d8c0f63a72-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d3ce5f64ef1889a3856db9d8c0f63a72 columnFamilyName family 2024-12-12T06:12:34,742 DEBUG [StoreOpener-d3ce5f64ef1889a3856db9d8c0f63a72-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-12T06:12:34,743 INFO [StoreOpener-d3ce5f64ef1889a3856db9d8c0f63a72-1 {}] regionserver.HStore(327): Store=d3ce5f64ef1889a3856db9d8c0f63a72/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-12T06:12:34,743 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for d3ce5f64ef1889a3856db9d8c0f63a72 2024-12-12T06:12:34,745 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/d3ce5f64ef1889a3856db9d8c0f63a72 2024-12-12T06:12:34,745 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1733983953560.5d34bdb8a1414fb25a475fb97fc440b4. 2024-12-12T06:12:34,745 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened test,lll,1733983953560.5d34bdb8a1414fb25a475fb97fc440b4. 2024-12-12T06:12:34,745 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(132): Open test,mmm,1733983953560.d86220c39cb75f305e7ed4d4c9f1191c. 2024-12-12T06:12:34,746 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7752): Opening region: {ENCODED => d86220c39cb75f305e7ed4d4c9f1191c, NAME => 'test,mmm,1733983953560.d86220c39cb75f305e7ed4d4c9f1191c.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-12-12T06:12:34,746 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d86220c39cb75f305e7ed4d4c9f1191c 2024-12-12T06:12:34,746 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(898): Instantiated test,mmm,1733983953560.d86220c39cb75f305e7ed4d4c9f1191c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-12T06:12:34,746 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7794): checking encryption for d86220c39cb75f305e7ed4d4c9f1191c 2024-12-12T06:12:34,746 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7797): checking classloading for d86220c39cb75f305e7ed4d4c9f1191c 2024-12-12T06:12:34,747 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/d3ce5f64ef1889a3856db9d8c0f63a72 2024-12-12T06:12:34,747 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for d3ce5f64ef1889a3856db9d8c0f63a72 2024-12-12T06:12:34,748 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=12 2024-12-12T06:12:34,748 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for d3ce5f64ef1889a3856db9d8c0f63a72 2024-12-12T06:12:34,748 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure f2b6a6b11f14670aca9dccceb20d9630, server=ffbfd3107920,42275,1733983951224 in 235 msec 2024-12-12T06:12:34,750 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for d3ce5f64ef1889a3856db9d8c0f63a72 2024-12-12T06:12:34,750 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f2b6a6b11f14670aca9dccceb20d9630, ASSIGN in 524 msec 2024-12-12T06:12:34,752 INFO [StoreOpener-d86220c39cb75f305e7ed4d4c9f1191c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d86220c39cb75f305e7ed4d4c9f1191c 2024-12-12T06:12:34,752 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/16838c5df0ab4e3938ad84e9a080def9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-12T06:12:34,753 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened 16838c5df0ab4e3938ad84e9a080def9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62978326, jitterRate=-0.06154981255531311}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-12T06:12:34,753 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 16838c5df0ab4e3938ad84e9a080def9 2024-12-12T06:12:34,753 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45209 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=ffbfd3107920,33455,1733983951151, table=test, region=60c948f12e02b6ab0066426b6aceea80. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-12T06:12:34,753 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45209 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=ffbfd3107920,33455,1733983951151, table=test, region=ae428798a6433b4d48a205ab3d14ba42. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-12T06:12:34,753 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for 16838c5df0ab4e3938ad84e9a080def9: Running coprocessor pre-open hook at 1733983954724Writing region info on filesystem at 1733983954724Initializing all the Stores at 1733983954726 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733983954726Cleaning up temporary data from old regions at 1733983954737 (+11 ms)Running coprocessor post-open hooks at 1733983954753 (+16 ms)Region opened successfully at 1733983954753 2024-12-12T06:12:34,753 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=5d34bdb8a1414fb25a475fb97fc440b4, regionState=OPEN, openSeqNum=2, regionLocation=ffbfd3107920,33455,1733983951151 2024-12-12T06:12:34,756 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for test,bbb,1733983953560.16838c5df0ab4e3938ad84e9a080def9., pid=47, masterSystemTime=1733983954609 2024-12-12T06:12:34,756 INFO [StoreOpener-d86220c39cb75f305e7ed4d4c9f1191c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d86220c39cb75f305e7ed4d4c9f1191c columnFamilyName family 2024-12-12T06:12:34,756 DEBUG [StoreOpener-d86220c39cb75f305e7ed4d4c9f1191c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-12T06:12:34,757 INFO [StoreOpener-d86220c39cb75f305e7ed4d4c9f1191c-1 {}] regionserver.HStore(327): Store=d86220c39cb75f305e7ed4d4c9f1191c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-12T06:12:34,757 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1038): replaying wal for d86220c39cb75f305e7ed4d4c9f1191c 2024-12-12T06:12:34,757 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,33455,1733983951151 because future has completed 2024-12-12T06:12:34,758 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/d86220c39cb75f305e7ed4d4c9f1191c 2024-12-12T06:12:34,759 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for test,bbb,1733983953560.16838c5df0ab4e3938ad84e9a080def9. 2024-12-12T06:12:34,759 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/d86220c39cb75f305e7ed4d4c9f1191c 2024-12-12T06:12:34,759 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened test,bbb,1733983953560.16838c5df0ab4e3938ad84e9a080def9. 2024-12-12T06:12:34,759 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(132): Open test,eee,1733983953560.b935ee7eeea02b161c0ebc0d05527e91. 2024-12-12T06:12:34,760 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7752): Opening region: {ENCODED => b935ee7eeea02b161c0ebc0d05527e91, NAME => 'test,eee,1733983953560.b935ee7eeea02b161c0ebc0d05527e91.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-12-12T06:12:34,760 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test b935ee7eeea02b161c0ebc0d05527e91 2024-12-12T06:12:34,760 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(898): Instantiated test,eee,1733983953560.b935ee7eeea02b161c0ebc0d05527e91.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-12T06:12:34,760 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7794): checking encryption for b935ee7eeea02b161c0ebc0d05527e91 2024-12-12T06:12:34,760 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/d3ce5f64ef1889a3856db9d8c0f63a72/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-12T06:12:34,760 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7797): checking classloading for b935ee7eeea02b161c0ebc0d05527e91 2024-12-12T06:12:34,761 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1048): stopping wal replay for d86220c39cb75f305e7ed4d4c9f1191c 2024-12-12T06:12:34,761 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1060): Cleaning up temporary data for d86220c39cb75f305e7ed4d4c9f1191c 2024-12-12T06:12:34,761 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened d3ce5f64ef1889a3856db9d8c0f63a72; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61913646, jitterRate=-0.07741478085517883}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-12T06:12:34,761 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d3ce5f64ef1889a3856db9d8c0f63a72 2024-12-12T06:12:34,762 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for d3ce5f64ef1889a3856db9d8c0f63a72: Running coprocessor pre-open hook at 1733983954730Writing region info on filesystem at 1733983954730Initializing all the Stores at 1733983954731 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733983954732 (+1 ms)Cleaning up temporary data from old regions at 1733983954748 (+16 ms)Running coprocessor post-open hooks at 1733983954761 (+13 ms)Region opened successfully at 1733983954762 (+1 ms) 2024-12-12T06:12:34,763 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for test,uuu,1733983953560.d3ce5f64ef1889a3856db9d8c0f63a72., pid=37, masterSystemTime=1733983954553 2024-12-12T06:12:34,765 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1093): writing seq id for d86220c39cb75f305e7ed4d4c9f1191c 2024-12-12T06:12:34,765 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=15 2024-12-12T06:12:34,765 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure 782c52424de7d71cb43b8fbfe8e89f7f, server=ffbfd3107920,46199,1733983951048 in 352 msec 2024-12-12T06:12:34,766 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for test,uuu,1733983953560.d3ce5f64ef1889a3856db9d8c0f63a72. 2024-12-12T06:12:34,766 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened test,uuu,1733983953560.d3ce5f64ef1889a3856db9d8c0f63a72. 2024-12-12T06:12:34,766 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open test,xxx,1733983953560.269d56dd3a6671f5c9a30401b73ade89. 2024-12-12T06:12:34,766 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => 269d56dd3a6671f5c9a30401b73ade89, NAME => 'test,xxx,1733983953560.269d56dd3a6671f5c9a30401b73ade89.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-12-12T06:12:34,767 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 269d56dd3a6671f5c9a30401b73ade89 2024-12-12T06:12:34,767 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated test,xxx,1733983953560.269d56dd3a6671f5c9a30401b73ade89.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-12T06:12:34,767 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for 269d56dd3a6671f5c9a30401b73ade89 2024-12-12T06:12:34,767 INFO [StoreOpener-b935ee7eeea02b161c0ebc0d05527e91-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region b935ee7eeea02b161c0ebc0d05527e91 2024-12-12T06:12:34,767 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for 269d56dd3a6671f5c9a30401b73ade89 2024-12-12T06:12:34,769 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=d3ce5f64ef1889a3856db9d8c0f63a72, regionState=OPEN, openSeqNum=2, regionLocation=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:34,769 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/d86220c39cb75f305e7ed4d4c9f1191c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-12T06:12:34,770 INFO [StoreOpener-b935ee7eeea02b161c0ebc0d05527e91-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b935ee7eeea02b161c0ebc0d05527e91 columnFamilyName family 2024-12-12T06:12:34,770 DEBUG [StoreOpener-b935ee7eeea02b161c0ebc0d05527e91-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-12T06:12:34,770 INFO [StoreOpener-269d56dd3a6671f5c9a30401b73ade89-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 269d56dd3a6671f5c9a30401b73ade89 2024-12-12T06:12:34,770 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1114): Opened d86220c39cb75f305e7ed4d4c9f1191c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66215587, jitterRate=-0.013310864567756653}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-12T06:12:34,770 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d86220c39cb75f305e7ed4d4c9f1191c 2024-12-12T06:12:34,770 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1006): Region open journal for d86220c39cb75f305e7ed4d4c9f1191c: Running coprocessor pre-open hook at 1733983954746Writing region info on filesystem at 1733983954746Initializing all the Stores at 1733983954747 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733983954748 (+1 ms)Cleaning up temporary data from old regions at 1733983954761 (+13 ms)Running coprocessor post-open hooks at 1733983954770 (+9 ms)Region opened successfully at 1733983954770 2024-12-12T06:12:34,771 INFO [StoreOpener-b935ee7eeea02b161c0ebc0d05527e91-1 {}] regionserver.HStore(327): Store=b935ee7eeea02b161c0ebc0d05527e91/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-12T06:12:34,771 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1038): replaying wal for b935ee7eeea02b161c0ebc0d05527e91 2024-12-12T06:12:34,771 INFO [StoreOpener-269d56dd3a6671f5c9a30401b73ade89-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 269d56dd3a6671f5c9a30401b73ade89 columnFamilyName family 2024-12-12T06:12:34,771 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1733983953560.d86220c39cb75f305e7ed4d4c9f1191c., pid=54, masterSystemTime=1733983954640 2024-12-12T06:12:34,772 DEBUG [StoreOpener-269d56dd3a6671f5c9a30401b73ade89-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-12T06:12:34,772 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/b935ee7eeea02b161c0ebc0d05527e91 2024-12-12T06:12:34,772 INFO [StoreOpener-269d56dd3a6671f5c9a30401b73ade89-1 {}] regionserver.HStore(327): Store=269d56dd3a6671f5c9a30401b73ade89/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-12T06:12:34,773 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for 269d56dd3a6671f5c9a30401b73ade89 2024-12-12T06:12:34,773 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/b935ee7eeea02b161c0ebc0d05527e91 2024-12-12T06:12:34,773 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure d3ce5f64ef1889a3856db9d8c0f63a72, server=ffbfd3107920,46199,1733983951048 because future has completed 2024-12-12T06:12:34,774 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/269d56dd3a6671f5c9a30401b73ade89 2024-12-12T06:12:34,774 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1048): stopping wal replay for b935ee7eeea02b161c0ebc0d05527e91 2024-12-12T06:12:34,775 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1060): Cleaning up temporary data for b935ee7eeea02b161c0ebc0d05527e91 2024-12-12T06:12:34,775 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/269d56dd3a6671f5c9a30401b73ade89 2024-12-12T06:12:34,775 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for 269d56dd3a6671f5c9a30401b73ade89 2024-12-12T06:12:34,775 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for 269d56dd3a6671f5c9a30401b73ade89 2024-12-12T06:12:34,775 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1733983953560.d86220c39cb75f305e7ed4d4c9f1191c. 2024-12-12T06:12:34,776 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(153): Opened test,mmm,1733983953560.d86220c39cb75f305e7ed4d4c9f1191c. 2024-12-12T06:12:34,776 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(132): Open test,sss,1733983953560.145c6ba0291376a96e3a37f6b274f63b. 2024-12-12T06:12:34,776 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7752): Opening region: {ENCODED => 145c6ba0291376a96e3a37f6b274f63b, NAME => 'test,sss,1733983953560.145c6ba0291376a96e3a37f6b274f63b.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-12-12T06:12:34,776 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 145c6ba0291376a96e3a37f6b274f63b 2024-12-12T06:12:34,776 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(898): Instantiated test,sss,1733983953560.145c6ba0291376a96e3a37f6b274f63b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-12T06:12:34,777 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7794): checking encryption for 145c6ba0291376a96e3a37f6b274f63b 2024-12-12T06:12:34,777 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7797): checking classloading for 145c6ba0291376a96e3a37f6b274f63b 2024-12-12T06:12:34,778 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1093): writing seq id for b935ee7eeea02b161c0ebc0d05527e91 2024-12-12T06:12:34,778 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for 269d56dd3a6671f5c9a30401b73ade89 2024-12-12T06:12:34,780 INFO [StoreOpener-145c6ba0291376a96e3a37f6b274f63b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 145c6ba0291376a96e3a37f6b274f63b 2024-12-12T06:12:34,782 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/269d56dd3a6671f5c9a30401b73ade89/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-12T06:12:34,782 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=25 2024-12-12T06:12:34,783 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure d3ce5f64ef1889a3856db9d8c0f63a72, server=ffbfd3107920,46199,1733983951048 in 337 msec 2024-12-12T06:12:34,782 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened 269d56dd3a6671f5c9a30401b73ade89; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72259254, jitterRate=0.07674679160118103}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-12T06:12:34,783 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 269d56dd3a6671f5c9a30401b73ade89 2024-12-12T06:12:34,783 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for 269d56dd3a6671f5c9a30401b73ade89: Running coprocessor pre-open hook at 1733983954767Writing region info on filesystem at 1733983954767Initializing all the Stores at 1733983954769 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733983954769Cleaning up temporary data from old regions at 1733983954775 (+6 ms)Running coprocessor post-open hooks at 1733983954783 (+8 ms)Region opened successfully at 1733983954783 2024-12-12T06:12:34,784 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/b935ee7eeea02b161c0ebc0d05527e91/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-12T06:12:34,784 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1733983953560.269d56dd3a6671f5c9a30401b73ade89., pid=31, masterSystemTime=1733983954553 2024-12-12T06:12:34,785 INFO [StoreOpener-145c6ba0291376a96e3a37f6b274f63b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 145c6ba0291376a96e3a37f6b274f63b columnFamilyName family 2024-12-12T06:12:34,785 DEBUG [StoreOpener-145c6ba0291376a96e3a37f6b274f63b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-12T06:12:34,786 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1114): Opened b935ee7eeea02b161c0ebc0d05527e91; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67960796, jitterRate=0.012694776058197021}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-12T06:12:34,786 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b935ee7eeea02b161c0ebc0d05527e91 2024-12-12T06:12:34,786 INFO [StoreOpener-145c6ba0291376a96e3a37f6b274f63b-1 {}] regionserver.HStore(327): Store=145c6ba0291376a96e3a37f6b274f63b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-12T06:12:34,786 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d3ce5f64ef1889a3856db9d8c0f63a72, ASSIGN in 559 msec 2024-12-12T06:12:34,786 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1006): Region open journal for b935ee7eeea02b161c0ebc0d05527e91: Running coprocessor pre-open hook at 1733983954761Writing region info on filesystem at 1733983954761Initializing all the Stores at 1733983954763 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733983954763Cleaning up temporary data from old regions at 1733983954775 (+12 ms)Running coprocessor post-open hooks at 1733983954786 (+11 ms)Region opened successfully at 1733983954786 2024-12-12T06:12:34,786 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1038): replaying wal for 145c6ba0291376a96e3a37f6b274f63b 2024-12-12T06:12:34,787 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1733983953560.b935ee7eeea02b161c0ebc0d05527e91., pid=48, masterSystemTime=1733983954609 2024-12-12T06:12:34,788 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/145c6ba0291376a96e3a37f6b274f63b 2024-12-12T06:12:34,788 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1733983953560.269d56dd3a6671f5c9a30401b73ade89. 2024-12-12T06:12:34,788 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened test,xxx,1733983953560.269d56dd3a6671f5c9a30401b73ade89. 2024-12-12T06:12:34,788 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/145c6ba0291376a96e3a37f6b274f63b 2024-12-12T06:12:34,789 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=d86220c39cb75f305e7ed4d4c9f1191c, regionState=OPEN, openSeqNum=2, regionLocation=ffbfd3107920,33455,1733983951151 2024-12-12T06:12:34,789 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1048): stopping wal replay for 145c6ba0291376a96e3a37f6b274f63b 2024-12-12T06:12:34,789 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1060): Cleaning up temporary data for 145c6ba0291376a96e3a37f6b274f63b 2024-12-12T06:12:34,791 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1093): writing seq id for 145c6ba0291376a96e3a37f6b274f63b 2024-12-12T06:12:34,791 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1733983953560.b935ee7eeea02b161c0ebc0d05527e91. 2024-12-12T06:12:34,792 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(153): Opened test,eee,1733983953560.b935ee7eeea02b161c0ebc0d05527e91. 2024-12-12T06:12:34,792 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(132): Open test,nnn,1733983953560.be4cb5da32eefc543e09e80b426b6938. 2024-12-12T06:12:34,792 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7752): Opening region: {ENCODED => be4cb5da32eefc543e09e80b426b6938, NAME => 'test,nnn,1733983953560.be4cb5da32eefc543e09e80b426b6938.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-12-12T06:12:34,792 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure d86220c39cb75f305e7ed4d4c9f1191c, server=ffbfd3107920,33455,1733983951151 because future has completed 2024-12-12T06:12:34,793 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test be4cb5da32eefc543e09e80b426b6938 2024-12-12T06:12:34,793 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(898): Instantiated test,nnn,1733983953560.be4cb5da32eefc543e09e80b426b6938.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-12T06:12:34,793 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7794): checking encryption for be4cb5da32eefc543e09e80b426b6938 2024-12-12T06:12:34,793 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7797): checking classloading for be4cb5da32eefc543e09e80b426b6938 2024-12-12T06:12:34,795 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=b935ee7eeea02b161c0ebc0d05527e91, regionState=OPEN, openSeqNum=2, regionLocation=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:34,795 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/145c6ba0291376a96e3a37f6b274f63b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-12T06:12:34,797 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1114): Opened 145c6ba0291376a96e3a37f6b274f63b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67455651, jitterRate=0.005167528986930847}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-12T06:12:34,797 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 145c6ba0291376a96e3a37f6b274f63b 2024-12-12T06:12:34,797 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1006): Region open journal for 145c6ba0291376a96e3a37f6b274f63b: Running coprocessor pre-open hook at 1733983954777Writing region info on filesystem at 1733983954777Initializing all the Stores at 1733983954778 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733983954778Cleaning up temporary data from old regions at 1733983954789 (+11 ms)Running coprocessor post-open hooks at 1733983954797 (+8 ms)Region opened successfully at 1733983954797 2024-12-12T06:12:34,798 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1733983953560.145c6ba0291376a96e3a37f6b274f63b., pid=50, masterSystemTime=1733983954640 2024-12-12T06:12:34,799 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=48, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure b935ee7eeea02b161c0ebc0d05527e91, server=ffbfd3107920,42275,1733983951224 because future has completed 2024-12-12T06:12:34,801 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1733983953560.145c6ba0291376a96e3a37f6b274f63b. 2024-12-12T06:12:34,801 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(153): Opened test,sss,1733983953560.145c6ba0291376a96e3a37f6b274f63b. 2024-12-12T06:12:34,801 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open test,ppp,1733983953560.1202ee0cad3e428893fcdbc630d4f9b9. 2024-12-12T06:12:34,802 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => 1202ee0cad3e428893fcdbc630d4f9b9, NAME => 'test,ppp,1733983953560.1202ee0cad3e428893fcdbc630d4f9b9.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-12-12T06:12:34,802 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 1202ee0cad3e428893fcdbc630d4f9b9 2024-12-12T06:12:34,802 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated test,ppp,1733983953560.1202ee0cad3e428893fcdbc630d4f9b9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-12T06:12:34,802 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for 1202ee0cad3e428893fcdbc630d4f9b9 2024-12-12T06:12:34,802 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for 1202ee0cad3e428893fcdbc630d4f9b9 2024-12-12T06:12:34,803 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=17 2024-12-12T06:12:34,803 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure d86220c39cb75f305e7ed4d4c9f1191c, server=ffbfd3107920,33455,1733983951151 in 269 msec 2024-12-12T06:12:34,806 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d86220c39cb75f305e7ed4d4c9f1191c, ASSIGN in 580 msec 2024-12-12T06:12:34,807 INFO [StoreOpener-be4cb5da32eefc543e09e80b426b6938-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region be4cb5da32eefc543e09e80b426b6938 2024-12-12T06:12:34,808 INFO [StoreOpener-1202ee0cad3e428893fcdbc630d4f9b9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 1202ee0cad3e428893fcdbc630d4f9b9 2024-12-12T06:12:34,809 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=145c6ba0291376a96e3a37f6b274f63b, regionState=OPEN, openSeqNum=2, regionLocation=ffbfd3107920,33455,1733983951151 2024-12-12T06:12:34,810 INFO [StoreOpener-be4cb5da32eefc543e09e80b426b6938-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region be4cb5da32eefc543e09e80b426b6938 columnFamilyName family 2024-12-12T06:12:34,810 DEBUG [StoreOpener-be4cb5da32eefc543e09e80b426b6938-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-12T06:12:34,811 INFO [StoreOpener-be4cb5da32eefc543e09e80b426b6938-1 {}] regionserver.HStore(327): Store=be4cb5da32eefc543e09e80b426b6938/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-12T06:12:34,811 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1038): replaying wal for be4cb5da32eefc543e09e80b426b6938 2024-12-12T06:12:34,812 INFO [StoreOpener-1202ee0cad3e428893fcdbc630d4f9b9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1202ee0cad3e428893fcdbc630d4f9b9 columnFamilyName family 2024-12-12T06:12:34,812 DEBUG [StoreOpener-1202ee0cad3e428893fcdbc630d4f9b9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-12T06:12:34,813 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/be4cb5da32eefc543e09e80b426b6938 2024-12-12T06:12:34,813 INFO [StoreOpener-1202ee0cad3e428893fcdbc630d4f9b9-1 {}] regionserver.HStore(327): Store=1202ee0cad3e428893fcdbc630d4f9b9/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-12T06:12:34,814 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for 1202ee0cad3e428893fcdbc630d4f9b9 2024-12-12T06:12:34,814 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/be4cb5da32eefc543e09e80b426b6938 2024-12-12T06:12:34,814 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 145c6ba0291376a96e3a37f6b274f63b, server=ffbfd3107920,33455,1733983951151 because future has completed 2024-12-12T06:12:34,814 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1048): stopping wal replay for be4cb5da32eefc543e09e80b426b6938 2024-12-12T06:12:34,814 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1060): Cleaning up temporary data for be4cb5da32eefc543e09e80b426b6938 2024-12-12T06:12:34,815 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/1202ee0cad3e428893fcdbc630d4f9b9 2024-12-12T06:12:34,816 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/1202ee0cad3e428893fcdbc630d4f9b9 2024-12-12T06:12:34,817 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for 1202ee0cad3e428893fcdbc630d4f9b9 2024-12-12T06:12:34,817 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1093): writing seq id for be4cb5da32eefc543e09e80b426b6938 2024-12-12T06:12:34,817 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for 1202ee0cad3e428893fcdbc630d4f9b9 2024-12-12T06:12:34,821 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=9 2024-12-12T06:12:34,821 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure b935ee7eeea02b161c0ebc0d05527e91, server=ffbfd3107920,42275,1733983951224 in 315 msec 2024-12-12T06:12:34,821 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/be4cb5da32eefc543e09e80b426b6938/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-12T06:12:34,821 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for 1202ee0cad3e428893fcdbc630d4f9b9 2024-12-12T06:12:34,822 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1114): Opened be4cb5da32eefc543e09e80b426b6938; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67908217, jitterRate=0.011911287903785706}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-12T06:12:34,822 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1122): Running coprocessor post-open hooks for be4cb5da32eefc543e09e80b426b6938 2024-12-12T06:12:34,822 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1006): Region open journal for be4cb5da32eefc543e09e80b426b6938: Running coprocessor pre-open hook at 1733983954793Writing region info on filesystem at 1733983954793Initializing all the Stores at 1733983954794 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733983954794Cleaning up temporary data from old regions at 1733983954814 (+20 ms)Running coprocessor post-open hooks at 1733983954822 (+8 ms)Region opened successfully at 1733983954822 2024-12-12T06:12:34,824 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2236): Post open deploy tasks for test,nnn,1733983953560.be4cb5da32eefc543e09e80b426b6938., pid=44, masterSystemTime=1733983954609 2024-12-12T06:12:34,825 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/1202ee0cad3e428893fcdbc630d4f9b9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-12T06:12:34,826 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=b935ee7eeea02b161c0ebc0d05527e91, ASSIGN in 597 msec 2024-12-12T06:12:34,826 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened 1202ee0cad3e428893fcdbc630d4f9b9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70767637, jitterRate=0.05451996624469757}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-12T06:12:34,827 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1202ee0cad3e428893fcdbc630d4f9b9 2024-12-12T06:12:34,827 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for 1202ee0cad3e428893fcdbc630d4f9b9: Running coprocessor pre-open hook at 1733983954802Writing region info on filesystem at 1733983954802Initializing all the Stores at 1733983954804 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733983954804Cleaning up temporary data from old regions at 1733983954817 (+13 ms)Running coprocessor post-open hooks at 1733983954827 (+10 ms)Region opened successfully at 1733983954827 2024-12-12T06:12:34,827 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2266): Finished post open deploy task for test,nnn,1733983953560.be4cb5da32eefc543e09e80b426b6938. 2024-12-12T06:12:34,827 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(153): Opened test,nnn,1733983953560.be4cb5da32eefc543e09e80b426b6938. 2024-12-12T06:12:34,827 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open test,www,1733983953560.764fc3b5431e53dfb445323d0bfd39a6. 2024-12-12T06:12:34,828 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => 764fc3b5431e53dfb445323d0bfd39a6, NAME => 'test,www,1733983953560.764fc3b5431e53dfb445323d0bfd39a6.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-12-12T06:12:34,828 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1733983953560.1202ee0cad3e428893fcdbc630d4f9b9., pid=55, masterSystemTime=1733983954640 2024-12-12T06:12:34,828 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 764fc3b5431e53dfb445323d0bfd39a6 2024-12-12T06:12:34,829 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated test,www,1733983953560.764fc3b5431e53dfb445323d0bfd39a6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-12T06:12:34,829 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for 764fc3b5431e53dfb445323d0bfd39a6 2024-12-12T06:12:34,829 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for 764fc3b5431e53dfb445323d0bfd39a6 2024-12-12T06:12:34,831 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ppp,1733983953560.1202ee0cad3e428893fcdbc630d4f9b9. 2024-12-12T06:12:34,832 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened test,ppp,1733983953560.1202ee0cad3e428893fcdbc630d4f9b9. 2024-12-12T06:12:34,832 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(132): Open test,,1733983953560.8545c1fe97252abe3447dcfaebd5cfa5. 2024-12-12T06:12:34,832 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7752): Opening region: {ENCODED => 8545c1fe97252abe3447dcfaebd5cfa5, NAME => 'test,,1733983953560.8545c1fe97252abe3447dcfaebd5cfa5.', STARTKEY => '', ENDKEY => 'bbb'} 2024-12-12T06:12:34,832 INFO [StoreOpener-764fc3b5431e53dfb445323d0bfd39a6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 764fc3b5431e53dfb445323d0bfd39a6 2024-12-12T06:12:34,832 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8545c1fe97252abe3447dcfaebd5cfa5 2024-12-12T06:12:34,833 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(898): Instantiated test,,1733983953560.8545c1fe97252abe3447dcfaebd5cfa5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-12T06:12:34,833 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7794): checking encryption for 8545c1fe97252abe3447dcfaebd5cfa5 2024-12-12T06:12:34,833 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7797): checking classloading for 8545c1fe97252abe3447dcfaebd5cfa5 2024-12-12T06:12:34,836 INFO [StoreOpener-8545c1fe97252abe3447dcfaebd5cfa5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8545c1fe97252abe3447dcfaebd5cfa5 2024-12-12T06:12:34,836 INFO [StoreOpener-764fc3b5431e53dfb445323d0bfd39a6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 764fc3b5431e53dfb445323d0bfd39a6 columnFamilyName family 2024-12-12T06:12:34,836 DEBUG [StoreOpener-764fc3b5431e53dfb445323d0bfd39a6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-12T06:12:34,836 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=23 2024-12-12T06:12:34,836 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 145c6ba0291376a96e3a37f6b274f63b, server=ffbfd3107920,33455,1733983951151 in 318 msec 2024-12-12T06:12:34,837 INFO [StoreOpener-764fc3b5431e53dfb445323d0bfd39a6-1 {}] regionserver.HStore(327): Store=764fc3b5431e53dfb445323d0bfd39a6/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-12T06:12:34,837 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for 764fc3b5431e53dfb445323d0bfd39a6 2024-12-12T06:12:34,838 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/764fc3b5431e53dfb445323d0bfd39a6 2024-12-12T06:12:34,839 INFO [StoreOpener-8545c1fe97252abe3447dcfaebd5cfa5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8545c1fe97252abe3447dcfaebd5cfa5 columnFamilyName family 2024-12-12T06:12:34,839 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN in 612 msec 2024-12-12T06:12:34,839 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/764fc3b5431e53dfb445323d0bfd39a6 2024-12-12T06:12:34,839 DEBUG [StoreOpener-8545c1fe97252abe3447dcfaebd5cfa5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-12T06:12:34,840 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for 764fc3b5431e53dfb445323d0bfd39a6 2024-12-12T06:12:34,840 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for 764fc3b5431e53dfb445323d0bfd39a6 2024-12-12T06:12:34,840 INFO [StoreOpener-8545c1fe97252abe3447dcfaebd5cfa5-1 {}] regionserver.HStore(327): Store=8545c1fe97252abe3447dcfaebd5cfa5/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-12T06:12:34,841 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1038): replaying wal for 8545c1fe97252abe3447dcfaebd5cfa5 2024-12-12T06:12:34,841 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=1202ee0cad3e428893fcdbc630d4f9b9, regionState=OPEN, openSeqNum=2, regionLocation=ffbfd3107920,33455,1733983951151 2024-12-12T06:12:34,842 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/8545c1fe97252abe3447dcfaebd5cfa5 2024-12-12T06:12:34,843 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/8545c1fe97252abe3447dcfaebd5cfa5 2024-12-12T06:12:34,843 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1048): stopping wal replay for 8545c1fe97252abe3447dcfaebd5cfa5 2024-12-12T06:12:34,843 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for 764fc3b5431e53dfb445323d0bfd39a6 2024-12-12T06:12:34,843 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1060): Cleaning up temporary data for 8545c1fe97252abe3447dcfaebd5cfa5 2024-12-12T06:12:34,845 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45209 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=ffbfd3107920,46199,1733983951048, table=test, region=2eeaafb9a1964f4c6899fa5dde05a463. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-12T06:12:34,845 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45209 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=ffbfd3107920,46199,1733983951048, table=test, region=c98bb859cc9e047b211ddb4c5b967ce5. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-12T06:12:34,845 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1202ee0cad3e428893fcdbc630d4f9b9, server=ffbfd3107920,33455,1733983951151 because future has completed 2024-12-12T06:12:34,846 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1093): writing seq id for 8545c1fe97252abe3447dcfaebd5cfa5 2024-12-12T06:12:34,849 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/764fc3b5431e53dfb445323d0bfd39a6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-12T06:12:34,850 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=be4cb5da32eefc543e09e80b426b6938, regionState=OPEN, openSeqNum=2, regionLocation=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:34,850 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/8545c1fe97252abe3447dcfaebd5cfa5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-12T06:12:34,850 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened 764fc3b5431e53dfb445323d0bfd39a6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74472244, jitterRate=0.10972291231155396}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-12T06:12:34,850 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 764fc3b5431e53dfb445323d0bfd39a6 2024-12-12T06:12:34,850 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for 764fc3b5431e53dfb445323d0bfd39a6: Running coprocessor pre-open hook at 1733983954829Writing region info on filesystem at 1733983954829Initializing all the Stores at 1733983954831 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733983954831Cleaning up temporary data from old regions at 1733983954840 (+9 ms)Running coprocessor post-open hooks at 1733983954850 (+10 ms)Region opened successfully at 1733983954850 2024-12-12T06:12:34,851 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1114): Opened 8545c1fe97252abe3447dcfaebd5cfa5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60882386, jitterRate=-0.09278175234794617}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-12T06:12:34,851 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8545c1fe97252abe3447dcfaebd5cfa5 2024-12-12T06:12:34,851 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1006): Region open journal for 8545c1fe97252abe3447dcfaebd5cfa5: Running coprocessor pre-open hook at 1733983954833Writing region info on filesystem at 1733983954833Initializing all the Stores at 1733983954835 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733983954835Cleaning up temporary data from old regions at 1733983954843 (+8 ms)Running coprocessor post-open hooks at 1733983954851 (+8 ms)Region opened successfully at 1733983954851 2024-12-12T06:12:34,852 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for test,www,1733983953560.764fc3b5431e53dfb445323d0bfd39a6., pid=41, masterSystemTime=1733983954609 2024-12-12T06:12:34,852 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1733983953560.8545c1fe97252abe3447dcfaebd5cfa5., pid=56, masterSystemTime=1733983954640 2024-12-12T06:12:34,855 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure be4cb5da32eefc543e09e80b426b6938, server=ffbfd3107920,42275,1733983951224 because future has completed 2024-12-12T06:12:34,855 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2266): Finished post open deploy task for test,,1733983953560.8545c1fe97252abe3447dcfaebd5cfa5. 2024-12-12T06:12:34,855 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(153): Opened test,,1733983953560.8545c1fe97252abe3447dcfaebd5cfa5. 2024-12-12T06:12:34,855 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(132): Open test,iii,1733983953560.57995ad3e527ada56b846564833341d1. 2024-12-12T06:12:34,855 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7752): Opening region: {ENCODED => 57995ad3e527ada56b846564833341d1, NAME => 'test,iii,1733983953560.57995ad3e527ada56b846564833341d1.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-12-12T06:12:34,856 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 57995ad3e527ada56b846564833341d1 2024-12-12T06:12:34,856 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(898): Instantiated test,iii,1733983953560.57995ad3e527ada56b846564833341d1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-12T06:12:34,856 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7794): checking encryption for 57995ad3e527ada56b846564833341d1 2024-12-12T06:12:34,856 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7797): checking classloading for 57995ad3e527ada56b846564833341d1 2024-12-12T06:12:34,857 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for test,www,1733983953560.764fc3b5431e53dfb445323d0bfd39a6. 2024-12-12T06:12:34,857 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened test,www,1733983953560.764fc3b5431e53dfb445323d0bfd39a6. 2024-12-12T06:12:34,858 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45209 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=ffbfd3107920,33455,1733983951151, table=test, region=5d34bdb8a1414fb25a475fb97fc440b4. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-12T06:12:34,858 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45209 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=ffbfd3107920,33455,1733983951151, table=test, region=60c948f12e02b6ab0066426b6aceea80. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-12T06:12:34,858 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45209 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=ffbfd3107920,33455,1733983951151, table=test, region=ae428798a6433b4d48a205ab3d14ba42. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-12T06:12:34,859 INFO [StoreOpener-57995ad3e527ada56b846564833341d1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 57995ad3e527ada56b846564833341d1 2024-12-12T06:12:34,860 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=20 2024-12-12T06:12:34,860 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 1202ee0cad3e428893fcdbc630d4f9b9, server=ffbfd3107920,33455,1733983951151 in 319 msec 2024-12-12T06:12:34,862 INFO [StoreOpener-57995ad3e527ada56b846564833341d1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 57995ad3e527ada56b846564833341d1 columnFamilyName family 2024-12-12T06:12:34,862 DEBUG [StoreOpener-57995ad3e527ada56b846564833341d1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-12T06:12:34,863 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=1202ee0cad3e428893fcdbc630d4f9b9, ASSIGN in 636 msec 2024-12-12T06:12:34,863 INFO [StoreOpener-57995ad3e527ada56b846564833341d1-1 {}] regionserver.HStore(327): Store=57995ad3e527ada56b846564833341d1/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-12T06:12:34,863 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1038): replaying wal for 57995ad3e527ada56b846564833341d1 2024-12-12T06:12:34,864 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=764fc3b5431e53dfb445323d0bfd39a6, regionState=OPEN, openSeqNum=2, regionLocation=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:34,864 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/57995ad3e527ada56b846564833341d1 2024-12-12T06:12:34,865 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/57995ad3e527ada56b846564833341d1 2024-12-12T06:12:34,866 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1048): stopping wal replay for 57995ad3e527ada56b846564833341d1 2024-12-12T06:12:34,866 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1060): Cleaning up temporary data for 57995ad3e527ada56b846564833341d1 2024-12-12T06:12:34,868 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 764fc3b5431e53dfb445323d0bfd39a6, server=ffbfd3107920,42275,1733983951224 because future has completed 2024-12-12T06:12:34,869 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): writing seq id for 57995ad3e527ada56b846564833341d1 2024-12-12T06:12:34,872 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/57995ad3e527ada56b846564833341d1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-12T06:12:34,872 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=18 2024-12-12T06:12:34,872 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure be4cb5da32eefc543e09e80b426b6938, server=ffbfd3107920,42275,1733983951224 in 388 msec 2024-12-12T06:12:34,873 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1114): Opened 57995ad3e527ada56b846564833341d1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65821539, jitterRate=-0.019182637333869934}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-12T06:12:34,873 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 57995ad3e527ada56b846564833341d1 2024-12-12T06:12:34,873 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1006): Region open journal for 57995ad3e527ada56b846564833341d1: Running coprocessor pre-open hook at 1733983954856Writing region info on filesystem at 1733983954856Initializing all the Stores at 1733983954858 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733983954858Cleaning up temporary data from old regions at 1733983954866 (+8 ms)Running coprocessor post-open hooks at 1733983954873 (+7 ms)Region opened successfully at 1733983954873 2024-12-12T06:12:34,874 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1733983953560.57995ad3e527ada56b846564833341d1., pid=51, masterSystemTime=1733983954640 2024-12-12T06:12:34,875 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=be4cb5da32eefc543e09e80b426b6938, ASSIGN in 649 msec 2024-12-12T06:12:34,877 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2266): Finished post open deploy task for test,iii,1733983953560.57995ad3e527ada56b846564833341d1. 2024-12-12T06:12:34,877 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(153): Opened test,iii,1733983953560.57995ad3e527ada56b846564833341d1. 2024-12-12T06:12:34,877 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(132): Open test,fff,1733983953560.4584d26a35c36968de829b34ac08fb5d. 2024-12-12T06:12:34,877 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7752): Opening region: {ENCODED => 4584d26a35c36968de829b34ac08fb5d, NAME => 'test,fff,1733983953560.4584d26a35c36968de829b34ac08fb5d.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-12-12T06:12:34,877 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4584d26a35c36968de829b34ac08fb5d 2024-12-12T06:12:34,877 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(898): Instantiated test,fff,1733983953560.4584d26a35c36968de829b34ac08fb5d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-12T06:12:34,877 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7794): checking encryption for 4584d26a35c36968de829b34ac08fb5d 2024-12-12T06:12:34,877 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7797): checking classloading for 4584d26a35c36968de829b34ac08fb5d 2024-12-12T06:12:34,879 INFO [StoreOpener-4584d26a35c36968de829b34ac08fb5d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4584d26a35c36968de829b34ac08fb5d 2024-12-12T06:12:34,881 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=27 2024-12-12T06:12:34,881 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure 764fc3b5431e53dfb445323d0bfd39a6, server=ffbfd3107920,42275,1733983951224 in 412 msec 2024-12-12T06:12:34,881 INFO [StoreOpener-4584d26a35c36968de829b34ac08fb5d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4584d26a35c36968de829b34ac08fb5d columnFamilyName family 2024-12-12T06:12:34,882 DEBUG [StoreOpener-4584d26a35c36968de829b34ac08fb5d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-12T06:12:34,882 INFO [StoreOpener-4584d26a35c36968de829b34ac08fb5d-1 {}] regionserver.HStore(327): Store=4584d26a35c36968de829b34ac08fb5d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-12T06:12:34,883 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1038): replaying wal for 4584d26a35c36968de829b34ac08fb5d 2024-12-12T06:12:34,884 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/4584d26a35c36968de829b34ac08fb5d 2024-12-12T06:12:34,884 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/4584d26a35c36968de829b34ac08fb5d 2024-12-12T06:12:34,885 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=764fc3b5431e53dfb445323d0bfd39a6, ASSIGN in 658 msec 2024-12-12T06:12:34,885 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1048): stopping wal replay for 4584d26a35c36968de829b34ac08fb5d 2024-12-12T06:12:34,885 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1060): Cleaning up temporary data for 4584d26a35c36968de829b34ac08fb5d 2024-12-12T06:12:34,886 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=57995ad3e527ada56b846564833341d1, regionState=OPEN, openSeqNum=2, regionLocation=ffbfd3107920,33455,1733983951151 2024-12-12T06:12:34,888 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): writing seq id for 4584d26a35c36968de829b34ac08fb5d 2024-12-12T06:12:34,889 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 57995ad3e527ada56b846564833341d1, server=ffbfd3107920,33455,1733983951151 because future has completed 2024-12-12T06:12:34,890 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=8545c1fe97252abe3447dcfaebd5cfa5, regionState=OPEN, openSeqNum=2, regionLocation=ffbfd3107920,33455,1733983951151 2024-12-12T06:12:34,893 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/4584d26a35c36968de829b34ac08fb5d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-12T06:12:34,894 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8545c1fe97252abe3447dcfaebd5cfa5, server=ffbfd3107920,33455,1733983951151 because future has completed 2024-12-12T06:12:34,893 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1114): Opened 4584d26a35c36968de829b34ac08fb5d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59396715, jitterRate=-0.11491997539997101}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-12T06:12:34,894 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4584d26a35c36968de829b34ac08fb5d 2024-12-12T06:12:34,894 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1006): Region open journal for 4584d26a35c36968de829b34ac08fb5d: Running coprocessor pre-open hook at 1733983954878Writing region info on filesystem at 1733983954878Initializing all the Stores at 1733983954879 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733983954879Cleaning up temporary data from old regions at 1733983954885 (+6 ms)Running coprocessor post-open hooks at 1733983954894 (+9 ms)Region opened successfully at 1733983954894 2024-12-12T06:12:34,895 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1733983953560.4584d26a35c36968de829b34ac08fb5d., pid=52, masterSystemTime=1733983954640 2024-12-12T06:12:34,898 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1733983953560.4584d26a35c36968de829b34ac08fb5d. 2024-12-12T06:12:34,898 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(153): Opened test,fff,1733983953560.4584d26a35c36968de829b34ac08fb5d. 2024-12-12T06:12:34,898 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=13 2024-12-12T06:12:34,898 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure 57995ad3e527ada56b846564833341d1, server=ffbfd3107920,33455,1733983951151 in 376 msec 2024-12-12T06:12:34,901 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN in 674 msec 2024-12-12T06:12:34,902 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=4584d26a35c36968de829b34ac08fb5d, regionState=OPEN, openSeqNum=2, regionLocation=ffbfd3107920,33455,1733983951151 2024-12-12T06:12:34,905 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4584d26a35c36968de829b34ac08fb5d, server=ffbfd3107920,33455,1733983951151 because future has completed 2024-12-12T06:12:34,909 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=5 2024-12-12T06:12:34,909 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 8545c1fe97252abe3447dcfaebd5cfa5, server=ffbfd3107920,33455,1733983951151 in 364 msec 2024-12-12T06:12:34,912 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8545c1fe97252abe3447dcfaebd5cfa5, ASSIGN in 685 msec 2024-12-12T06:12:34,917 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=10 2024-12-12T06:12:34,917 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 4584d26a35c36968de829b34ac08fb5d, server=ffbfd3107920,33455,1733983951151 in 390 msec 2024-12-12T06:12:34,920 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=4584d26a35c36968de829b34ac08fb5d, ASSIGN in 693 msec 2024-12-12T06:12:34,922 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=269d56dd3a6671f5c9a30401b73ade89, regionState=OPEN, openSeqNum=2, regionLocation=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:34,926 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 269d56dd3a6671f5c9a30401b73ade89, server=ffbfd3107920,46199,1733983951048 because future has completed 2024-12-12T06:12:34,928 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=782c52424de7d71cb43b8fbfe8e89f7f, ASSIGN in 701 msec 2024-12-12T06:12:34,933 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=28 2024-12-12T06:12:34,933 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=28, state=SUCCESS, hasLock=false; OpenRegionProcedure 269d56dd3a6671f5c9a30401b73ade89, server=ffbfd3107920,46199,1733983951048 in 532 msec 2024-12-12T06:12:34,937 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=269d56dd3a6671f5c9a30401b73ade89, ASSIGN in 710 msec 2024-12-12T06:12:34,938 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=16838c5df0ab4e3938ad84e9a080def9, regionState=OPEN, openSeqNum=2, regionLocation=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:34,941 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 16838c5df0ab4e3938ad84e9a080def9, server=ffbfd3107920,42275,1733983951224 because future has completed 2024-12-12T06:12:34,945 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=16 2024-12-12T06:12:34,946 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,33455,1733983951151 in 416 msec 2024-12-12T06:12:34,949 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN in 722 msec 2024-12-12T06:12:34,950 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45209 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=ffbfd3107920,46199,1733983951048, table=test, region=2eeaafb9a1964f4c6899fa5dde05a463. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-12T06:12:34,950 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45209 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=ffbfd3107920,46199,1733983951048, table=test, region=c98bb859cc9e047b211ddb4c5b967ce5. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-12T06:12:34,955 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=6 2024-12-12T06:12:34,955 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure 16838c5df0ab4e3938ad84e9a080def9, server=ffbfd3107920,42275,1733983951224 in 454 msec 2024-12-12T06:12:34,958 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=16838c5df0ab4e3938ad84e9a080def9, ASSIGN in 731 msec 2024-12-12T06:12:34,963 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45209 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=ffbfd3107920,33455,1733983951151, table=test, region=ae428798a6433b4d48a205ab3d14ba42. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-12T06:12:34,964 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=26 2024-12-12T06:12:34,964 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 60c948f12e02b6ab0066426b6aceea80, server=ffbfd3107920,33455,1733983951151 in 469 msec 2024-12-12T06:12:34,967 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=60c948f12e02b6ab0066426b6aceea80, ASSIGN in 740 msec 2024-12-12T06:12:34,971 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=7 2024-12-12T06:12:34,971 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 2eeaafb9a1964f4c6899fa5dde05a463, server=ffbfd3107920,46199,1733983951048 in 496 msec 2024-12-12T06:12:34,973 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=2eeaafb9a1964f4c6899fa5dde05a463, ASSIGN in 747 msec 2024-12-12T06:12:34,977 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=29 2024-12-12T06:12:34,977 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure ae428798a6433b4d48a205ab3d14ba42, server=ffbfd3107920,33455,1733983951151 in 488 msec 2024-12-12T06:12:34,980 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=ae428798a6433b4d48a205ab3d14ba42, ASSIGN in 754 msec 2024-12-12T06:12:34,984 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=8 2024-12-12T06:12:34,984 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure c98bb859cc9e047b211ddb4c5b967ce5, server=ffbfd3107920,46199,1733983951048 in 530 msec 2024-12-12T06:12:34,988 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=4 2024-12-12T06:12:34,988 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=c98bb859cc9e047b211ddb4c5b967ce5, ASSIGN in 760 msec 2024-12-12T06:12:34,990 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-12T06:12:34,991 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733983954990"}]},"ts":"1733983954990"} 2024-12-12T06:12:34,994 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLED in hbase:meta 2024-12-12T06:12:34,996 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_POST_OPERATION 2024-12-12T06:12:35,002 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test in 1.4280 sec 2024-12-12T06:12:35,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45209 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-12T06:12:35,747 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test completed 2024-12-12T06:12:35,747 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table test get assigned. Timeout = 60000ms 2024-12-12T06:12:35,749 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-12T06:12:35,773 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table test assigned to meta. Checking AM states. 2024-12-12T06:12:35,773 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-12T06:12:35,774 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table test assigned. 2024-12-12T06:12:35,774 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-12-12T06:12:35,781 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-12-12T06:12:35,807 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-12-12T06:12:35,826 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-12T06:12:35,828 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50992, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-12T06:12:35,830 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-12T06:12:35,833 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49300, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-12T06:12:35,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45209 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 8545c1fe97252abe3447dcfaebd5cfa5 changed from -1.0 to 0.0, refreshing cache 2024-12-12T06:12:35,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45209 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 4584d26a35c36968de829b34ac08fb5d changed from -1.0 to 0.0, refreshing cache 2024-12-12T06:12:35,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45209 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 57995ad3e527ada56b846564833341d1 changed from -1.0 to 0.0, refreshing cache 2024-12-12T06:12:35,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45209 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 5d34bdb8a1414fb25a475fb97fc440b4 changed from -1.0 to 0.0, refreshing cache 2024-12-12T06:12:35,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45209 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region d86220c39cb75f305e7ed4d4c9f1191c changed from -1.0 to 0.0, refreshing cache 2024-12-12T06:12:35,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45209 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1202ee0cad3e428893fcdbc630d4f9b9 changed from -1.0 to 0.0, refreshing cache 2024-12-12T06:12:35,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45209 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 145c6ba0291376a96e3a37f6b274f63b changed from -1.0 to 0.0, refreshing cache 2024-12-12T06:12:35,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45209 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 60c948f12e02b6ab0066426b6aceea80 changed from -1.0 to 0.0, refreshing cache 2024-12-12T06:12:35,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45209 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region ae428798a6433b4d48a205ab3d14ba42 changed from -1.0 to 0.0, refreshing cache 2024-12-12T06:12:35,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45209 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 16838c5df0ab4e3938ad84e9a080def9 changed from -1.0 to 0.0, refreshing cache 2024-12-12T06:12:35,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45209 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region b935ee7eeea02b161c0ebc0d05527e91 changed from -1.0 to 0.0, refreshing cache 2024-12-12T06:12:35,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45209 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f2b6a6b11f14670aca9dccceb20d9630 changed from -1.0 to 0.0, refreshing cache 2024-12-12T06:12:35,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45209 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region be4cb5da32eefc543e09e80b426b6938 changed from -1.0 to 0.0, refreshing cache 2024-12-12T06:12:35,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45209 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 653572572648db72daea94dd65cfb7f8 changed from -1.0 to 0.0, refreshing cache 2024-12-12T06:12:35,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45209 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 5162d3c0b3255050cbeed27e4456e5c2 changed from -1.0 to 0.0, refreshing cache 2024-12-12T06:12:35,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45209 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 764fc3b5431e53dfb445323d0bfd39a6 changed from -1.0 to 0.0, refreshing cache 2024-12-12T06:12:35,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45209 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 269eed2924060312949b51c10150f409 changed from -1.0 to 0.0, refreshing cache 2024-12-12T06:12:35,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45209 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 2eeaafb9a1964f4c6899fa5dde05a463 changed from -1.0 to 0.0, refreshing cache 2024-12-12T06:12:35,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45209 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c98bb859cc9e047b211ddb4c5b967ce5 changed from -1.0 to 0.0, refreshing cache 2024-12-12T06:12:35,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45209 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region b717f65da42dacb67935407f094e8467 changed from -1.0 to 0.0, refreshing cache 2024-12-12T06:12:35,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45209 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c77db308a7c1c57fe9ef945e17e28127 changed from -1.0 to 0.0, refreshing cache 2024-12-12T06:12:35,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45209 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 782c52424de7d71cb43b8fbfe8e89f7f changed from -1.0 to 0.0, refreshing cache 2024-12-12T06:12:35,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45209 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region a4bf848f8297d89de7b203fcc6678eb5 changed from -1.0 to 0.0, refreshing cache 2024-12-12T06:12:35,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45209 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f5fe5dc5eb9193cba5339ee508dff867 changed from -1.0 to 0.0, refreshing cache 2024-12-12T06:12:35,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45209 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region d3ce5f64ef1889a3856db9d8c0f63a72 changed from -1.0 to 0.0, refreshing cache 2024-12-12T06:12:35,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45209 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 269d56dd3a6671f5c9a30401b73ade89 changed from -1.0 to 0.0, refreshing cache 2024-12-12T06:12:35,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45209 {}] rsgroup.RSGroupBasedLoadBalancer(138): Balancing RSGroup=default 2024-12-12T06:12:35,892 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45209 {}] rsgroup.RSGroupBasedLoadBalancer(151): Start Generate Balance plan for group: default 2024-12-12T06:12:35,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45209 {}] balancer.BaseLoadBalancer(619): Start Generate Balance plan for cluster. 2024-12-12T06:12:35,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45209 {}] balancer.BalancerClusterState(204): Hosts are {ffbfd3107920=0} racks are {/default-rack=0} 2024-12-12T06:12:35,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45209 {}] balancer.BalancerClusterState(303): server 0 has 10 regions 2024-12-12T06:12:35,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45209 {}] balancer.BalancerClusterState(303): server 1 has 8 regions 2024-12-12T06:12:35,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45209 {}] balancer.BalancerClusterState(303): server 2 has 9 regions 2024-12-12T06:12:35,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45209 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-12T06:12:35,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45209 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-12T06:12:35,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45209 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-12T06:12:35,894 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45209 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-12T06:12:35,894 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45209 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-12T06:12:35,894 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45209 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-12T06:12:35,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45209 {}] balancer.BalancerClusterState(326): Number of tables=2, number of hosts=1, number of racks=1 2024-12-12T06:12:35,897 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45209 {}] balancer.StochasticLoadBalancer(403): Cluster wide - Calculating plan. may take up to 30000ms to complete. 2024-12-12T06:12:35,899 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45209 {}] balancer.StochasticLoadBalancer(515): Start StochasticLoadBalancer.balancer, initial weighted average imbalance=0.06931633435746164, functionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.06415002990995843, need balance); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.0); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); computedMaxSteps=64800 2024-12-12T06:12:36,286 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45209 {}] balancer.StochasticLoadBalancer(562): Finished computing new moving plan. Computation took 391 ms to try 64800 different iterations. Found a solution that moves 1 regions; Going from a computed imbalance of 0.06931633435746164 to a new imbalance of 0.01690158032826896. funtionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.0); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.037037037037037035, need balance); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); 2024-12-12T06:12:36,289 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45209 {}] master.HMaster(2167): Balancer plans size is 1, the balance interval is 300000 ms, and the max number regions in transition is 27 2024-12-12T06:12:36,289 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45209 {}] master.HMaster(2172): balance hri=60c948f12e02b6ab0066426b6aceea80, source=ffbfd3107920,33455,1733983951151, destination=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:36,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45209 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test, region=60c948f12e02b6ab0066426b6aceea80, REOPEN/MOVE 2024-12-12T06:12:36,291 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test, region=60c948f12e02b6ab0066426b6aceea80, REOPEN/MOVE 2024-12-12T06:12:36,293 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=60c948f12e02b6ab0066426b6aceea80, regionState=CLOSING, regionLocation=ffbfd3107920,33455,1733983951151 2024-12-12T06:12:36,295 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=test, region=60c948f12e02b6ab0066426b6aceea80, REOPEN/MOVE because future has completed 2024-12-12T06:12:36,296 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-12T06:12:36,296 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE, hasLock=false; CloseRegionProcedure 60c948f12e02b6ab0066426b6aceea80, server=ffbfd3107920,33455,1733983951151}] 2024-12-12T06:12:36,453 INFO [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(122): Close 60c948f12e02b6ab0066426b6aceea80 2024-12-12T06:12:36,453 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-12T06:12:36,454 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1722): Closing 60c948f12e02b6ab0066426b6aceea80, disabling compactions & flushes 2024-12-12T06:12:36,454 INFO [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1755): Closing region test,vvv,1733983953560.60c948f12e02b6ab0066426b6aceea80. 2024-12-12T06:12:36,454 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1733983953560.60c948f12e02b6ab0066426b6aceea80. 2024-12-12T06:12:36,454 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1733983953560.60c948f12e02b6ab0066426b6aceea80. after waiting 0 ms 2024-12-12T06:12:36,454 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1733983953560.60c948f12e02b6ab0066426b6aceea80. 2024-12-12T06:12:36,459 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/60c948f12e02b6ab0066426b6aceea80/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-12T06:12:36,462 INFO [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1973): Closed test,vvv,1733983953560.60c948f12e02b6ab0066426b6aceea80. 2024-12-12T06:12:36,462 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1676): Region close journal for 60c948f12e02b6ab0066426b6aceea80: Waiting for close lock at 1733983956453Running coprocessor pre-close hooks at 1733983956453Disabling compacts and flushes for region at 1733983956453Disabling writes for close at 1733983956454 (+1 ms)Writing region close event to WAL at 1733983956454Running coprocessor post-close hooks at 1733983956460 (+6 ms)Closed at 1733983956462 (+2 ms) 2024-12-12T06:12:36,463 INFO [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegionServer(3302): Adding 60c948f12e02b6ab0066426b6aceea80 move to ffbfd3107920,42275,1733983951224 record at close sequenceid=2 2024-12-12T06:12:36,466 INFO [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(157): Closed 60c948f12e02b6ab0066426b6aceea80 2024-12-12T06:12:36,467 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=60c948f12e02b6ab0066426b6aceea80, regionState=CLOSED 2024-12-12T06:12:36,469 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE, hasLock=false; CloseRegionProcedure 60c948f12e02b6ab0066426b6aceea80, server=ffbfd3107920,33455,1733983951151 because future has completed 2024-12-12T06:12:36,473 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-12-12T06:12:36,473 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; CloseRegionProcedure 60c948f12e02b6ab0066426b6aceea80, server=ffbfd3107920,33455,1733983951151 in 174 msec 2024-12-12T06:12:36,474 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=60c948f12e02b6ab0066426b6aceea80, REOPEN/MOVE; state=CLOSED, location=ffbfd3107920,42275,1733983951224; forceNewPlan=false, retain=false 2024-12-12T06:12:36,624 INFO [ffbfd3107920:45209 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-12T06:12:36,625 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=60c948f12e02b6ab0066426b6aceea80, regionState=OPENING, regionLocation=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:36,629 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=60c948f12e02b6ab0066426b6aceea80, REOPEN/MOVE because future has completed 2024-12-12T06:12:36,630 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=57, state=RUNNABLE, hasLock=false; OpenRegionProcedure 60c948f12e02b6ab0066426b6aceea80, server=ffbfd3107920,42275,1733983951224}] 2024-12-12T06:12:36,790 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(132): Open test,vvv,1733983953560.60c948f12e02b6ab0066426b6aceea80. 2024-12-12T06:12:36,790 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7752): Opening region: {ENCODED => 60c948f12e02b6ab0066426b6aceea80, NAME => 'test,vvv,1733983953560.60c948f12e02b6ab0066426b6aceea80.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-12-12T06:12:36,790 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 60c948f12e02b6ab0066426b6aceea80 2024-12-12T06:12:36,790 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(898): Instantiated test,vvv,1733983953560.60c948f12e02b6ab0066426b6aceea80.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-12T06:12:36,791 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7794): checking encryption for 60c948f12e02b6ab0066426b6aceea80 2024-12-12T06:12:36,791 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7797): checking classloading for 60c948f12e02b6ab0066426b6aceea80 2024-12-12T06:12:36,793 INFO [StoreOpener-60c948f12e02b6ab0066426b6aceea80-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 60c948f12e02b6ab0066426b6aceea80 2024-12-12T06:12:36,794 INFO [StoreOpener-60c948f12e02b6ab0066426b6aceea80-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 60c948f12e02b6ab0066426b6aceea80 columnFamilyName family 2024-12-12T06:12:36,794 DEBUG [StoreOpener-60c948f12e02b6ab0066426b6aceea80-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-12T06:12:36,795 INFO [StoreOpener-60c948f12e02b6ab0066426b6aceea80-1 {}] regionserver.HStore(327): Store=60c948f12e02b6ab0066426b6aceea80/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-12T06:12:36,795 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1038): replaying wal for 60c948f12e02b6ab0066426b6aceea80 2024-12-12T06:12:36,796 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/60c948f12e02b6ab0066426b6aceea80 2024-12-12T06:12:36,798 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/60c948f12e02b6ab0066426b6aceea80 2024-12-12T06:12:36,798 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1048): stopping wal replay for 60c948f12e02b6ab0066426b6aceea80 2024-12-12T06:12:36,798 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1060): Cleaning up temporary data for 60c948f12e02b6ab0066426b6aceea80 2024-12-12T06:12:36,800 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1093): writing seq id for 60c948f12e02b6ab0066426b6aceea80 2024-12-12T06:12:36,802 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1114): Opened 60c948f12e02b6ab0066426b6aceea80; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70186440, jitterRate=0.045859456062316895}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-12T06:12:36,802 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 60c948f12e02b6ab0066426b6aceea80 2024-12-12T06:12:36,802 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1006): Region open journal for 60c948f12e02b6ab0066426b6aceea80: Running coprocessor pre-open hook at 1733983956791Writing region info on filesystem at 1733983956791Initializing all the Stores at 1733983956792 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733983956792Cleaning up temporary data from old regions at 1733983956798 (+6 ms)Running coprocessor post-open hooks at 1733983956802 (+4 ms)Region opened successfully at 1733983956802 2024-12-12T06:12:36,803 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1733983953560.60c948f12e02b6ab0066426b6aceea80., pid=59, masterSystemTime=1733983956784 2024-12-12T06:12:36,805 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1733983953560.60c948f12e02b6ab0066426b6aceea80. 2024-12-12T06:12:36,805 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(153): Opened test,vvv,1733983953560.60c948f12e02b6ab0066426b6aceea80. 2024-12-12T06:12:36,806 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=60c948f12e02b6ab0066426b6aceea80, regionState=OPEN, openSeqNum=5, regionLocation=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:36,809 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=57, state=RUNNABLE, hasLock=false; OpenRegionProcedure 60c948f12e02b6ab0066426b6aceea80, server=ffbfd3107920,42275,1733983951224 because future has completed 2024-12-12T06:12:36,814 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=57 2024-12-12T06:12:36,814 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=57, state=SUCCESS, hasLock=false; OpenRegionProcedure 60c948f12e02b6ab0066426b6aceea80, server=ffbfd3107920,42275,1733983951224 in 181 msec 2024-12-12T06:12:36,816 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=60c948f12e02b6ab0066426b6aceea80, REOPEN/MOVE in 525 msec 2024-12-12T06:12:36,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45209 {}] master.HMaster(2203): Balancer is going into sleep until next period in 300000ms 2024-12-12T06:12:36,894 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [900,000] milli-secs(wait.for.ratio=[1]) 2024-12-12T06:12:36,901 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45209 {}] master.MasterRpcServices(567): Client=jenkins//172.17.0.2 set balanceSwitch=false 2024-12-12T06:12:36,922 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.assignment.TestRollbackSCP#testFailAndRollback Thread=385, OpenFileDescriptor=643, MaxFileDescriptor=1048576, SystemLoadAverage=65, ProcessCount=11, AvailableMemoryMB=9148 2024-12-12T06:12:36,925 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table hbase:meta 2024-12-12T06:12:36,926 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=hbase:meta,,1 2024-12-12T06:12:36,926 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing ffbfd3107920,33455,1733983951151 2024-12-12T06:12:36,927 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server ffbfd3107920,33455,1733983951151: Simulated kill ***** 2024-12-12T06:12:36,927 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-12T06:12:36,928 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-12T06:12:36,930 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-12T06:12:36,931 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-12T06:12:36,931 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-12T06:12:36,931 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 340556280 }, "NonHeapMemoryUsage": { "committed": 154402816, "init": 7667712, "max": -1, "used": 152041848 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "ffbfd3107920" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "ffbfd3107920" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.Context": "regionserver", "tag.Hostname": "ffbfd3107920" } ] } 2024-12-12T06:12:36,933 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45209 {}] master.MasterRpcServices(700): ffbfd3107920,33455,1733983951151 reported a fatal error: ***** ABORTING region server ffbfd3107920,33455,1733983951151: Simulated kill ***** 2024-12-12T06:12:36,935 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ffbfd3107920,33455,1733983951151' ***** 2024-12-12T06:12:36,935 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-12-12T06:12:36,935 INFO [RS:1;ffbfd3107920:33455 {}] regionserver.HeapMemoryManager(224): Stopping 2024-12-12T06:12:36,936 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [15,000] milli-secs(wait.for.ratio=[1]) 2024-12-12T06:12:36,936 INFO [RS:1;ffbfd3107920:33455 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-12-12T06:12:36,936 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-12T06:12:36,936 INFO [RS:1;ffbfd3107920:33455 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-12-12T06:12:36,936 INFO [RS:1;ffbfd3107920:33455 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-12T06:12:36,936 INFO [RS:1;ffbfd3107920:33455 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;ffbfd3107920:33455. 2024-12-12T06:12:36,936 DEBUG [RS:1;ffbfd3107920:33455 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-12T06:12:36,940 DEBUG [RS:1;ffbfd3107920:33455 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-12T06:12:36,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38541 is added to blk_1073741836_1012 (size=33749) 2024-12-12T06:12:36,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42285 is added to blk_1073741836_1012 (size=33749) 2024-12-12T06:12:36,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43493 is added to blk_1073741836_1012 (size=33749) 2024-12-12T06:12:36,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42285 is added to blk_1073741835_1011 (size=2769) 2024-12-12T06:12:36,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38541 is added to blk_1073741835_1011 (size=2769) 2024-12-12T06:12:36,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43493 is added to blk_1073741835_1011 (size=2769) 2024-12-12T06:12:36,958 DEBUG [RS:1;ffbfd3107920:33455 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-12T06:12:36,958 INFO [RS:1;ffbfd3107920:33455 {}] regionserver.LeaseManager(133): Closed leases 2024-12-12T06:12:36,958 INFO [RS:1;ffbfd3107920:33455 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33455 2024-12-12T06:12:36,978 INFO [regionserver/ffbfd3107920:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-12T06:12:37,015 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45209-0x10018b511340000, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-12T06:12:37,015 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33455-0x10018b511340002, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ffbfd3107920,33455,1733983951151 2024-12-12T06:12:37,015 INFO [RS:1;ffbfd3107920:33455 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-12T06:12:37,027 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ffbfd3107920,33455,1733983951151] 2024-12-12T06:12:37,035 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ffbfd3107920,33455,1733983951151 already deleted, retry=false 2024-12-12T06:12:37,035 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of ffbfd3107920,33455,1733983951151 on ffbfd3107920,45209,1733983950336 2024-12-12T06:12:37,042 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true 2024-12-12T06:12:37,048 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=60 for ffbfd3107920,33455,1733983951151 (carryingMeta=true) ffbfd3107920,33455,1733983951151/CRASHED/regionCount=9/lock=java.util.concurrent.locks.ReentrantReadWriteLock@4f7ca2b2[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-12T06:12:37,048 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=60, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true 2024-12-12T06:12:37,050 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=60, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true, isMeta: true 2024-12-12T06:12:37,052 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/WALs/ffbfd3107920,33455,1733983951151-splitting 2024-12-12T06:12:37,053 INFO [PEWorker-1 {}] master.SplitWALManager(105): ffbfd3107920,33455,1733983951151 WAL count=1, meta=true 2024-12-12T06:12:37,056 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta}] 2024-12-12T06:12:37,062 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:37,064 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224}] 2024-12-12T06:12:37,127 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33455-0x10018b511340002, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-12T06:12:37,127 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33455-0x10018b511340002, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-12T06:12:37,127 INFO [RS:1;ffbfd3107920:33455 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-12T06:12:37,127 INFO [RS:1;ffbfd3107920:33455 {}] regionserver.HRegionServer(1031): Exiting; stopping=ffbfd3107920,33455,1733983951151; zookeeper connection closed. 2024-12-12T06:12:37,128 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@36444aa7 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@36444aa7 2024-12-12T06:12:37,137 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-12-12T06:12:37,228 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42275 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=62 2024-12-12T06:12:37,249 INFO [RS_LOG_REPLAY_OPS-regionserver/ffbfd3107920:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(299): Splitting hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/WALs/ffbfd3107920,33455,1733983951151-splitting/ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, size=33.0 K (33749bytes) 2024-12-12T06:12:37,250 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ffbfd3107920:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-12-12T06:12:37,250 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ffbfd3107920:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-12-12T06:12:37,250 INFO [RS_LOG_REPLAY_OPS-regionserver/ffbfd3107920:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/WALs/ffbfd3107920,33455,1733983951151-splitting/ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta 2024-12-12T06:12:37,255 INFO [RS_LOG_REPLAY_OPS-regionserver/ffbfd3107920:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/WALs/ffbfd3107920,33455,1733983951151-splitting/ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta after 3ms 2024-12-12T06:12:37,262 INFO [RS_LOG_REPLAY_OPS-regionserver/ffbfd3107920:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(310): Open hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/WALs/ffbfd3107920,33455,1733983951151-splitting/ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta took 14ms 2024-12-12T06:12:37,270 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ffbfd3107920:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-12-12T06:12:37,275 INFO [RS_LOG_REPLAY_OPS-regionserver/ffbfd3107920:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta.temp 2024-12-12T06:12:37,277 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ffbfd3107920:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/WALs/ffbfd3107920,33455,1733983951151-splitting/ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta so closing down 2024-12-12T06:12:37,278 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ffbfd3107920:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-12T06:12:37,281 INFO [RS_LOG_REPLAY_OPS-regionserver/ffbfd3107920:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta.temp 2024-12-12T06:12:37,284 INFO [RS_LOG_REPLAY_OPS-regionserver/ffbfd3107920:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.OutputSink(145): 3 split writer threads finished 2024-12-12T06:12:37,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38541 is added to blk_1073741864_1040 (size=33508) 2024-12-12T06:12:37,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42285 is added to blk_1073741864_1040 (size=33508) 2024-12-12T06:12:37,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43493 is added to blk_1073741864_1040 (size=33508) 2024-12-12T06:12:37,294 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta.temp (wrote 61 edits, skipped 0 edits in 2 ms) 2024-12-12T06:12:37,296 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta.temp to hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-12-12T06:12:37,299 INFO [RS_LOG_REPLAY_OPS-regionserver/ffbfd3107920:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(425): Processed 62 edits across 1 Regions in 33 ms; skipped=1; WAL=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/WALs/ffbfd3107920,33455,1733983951151-splitting/ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, size=33.0 K, length=33749, corrupted=false, cancelled=false 2024-12-12T06:12:37,299 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ffbfd3107920:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(428): Completed split of hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/WALs/ffbfd3107920,33455,1733983951151-splitting/ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, journal: Splitting hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/WALs/ffbfd3107920,33455,1733983951151-splitting/ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, size=33.0 K (33749bytes) at 1733983957249Finishing writing output for hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/WALs/ffbfd3107920,33455,1733983951151-splitting/ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta so closing down at 1733983957277 (+28 ms)Creating recovered edits writer path=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta.temp at 1733983957281 (+4 ms)3 split writer threads finished at 1733983957284 (+3 ms)Closed recovered edits writer path=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta.temp (wrote 61 edits, skipped 0 edits in 2 ms) at 1733983957294 (+10 ms)Rename recovered edits hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta.temp to hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 at 1733983957296 (+2 ms)Processed 62 edits across 1 Regions in 33 ms; skipped=1; WAL=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/WALs/ffbfd3107920,33455,1733983951151-splitting/ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, size=33.0 K, length=33749, corrupted=false, cancelled=false at 1733983957299 (+3 ms) 2024-12-12T06:12:37,300 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ffbfd3107920:0-0 {event_type=RS_LOG_REPLAY, pid=62}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/WALs/ffbfd3107920,33455,1733983951151-splitting/ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta 2024-12-12T06:12:37,301 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ffbfd3107920:0-0 {event_type=RS_LOG_REPLAY, pid=62}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=62 2024-12-12T06:12:37,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45209 {}] master.HMaster(4169): Remote procedure done, pid=62 2024-12-12T06:12:37,311 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/WALs/ffbfd3107920,33455,1733983951151-splitting/ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta to hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/oldWALs 2024-12-12T06:12:37,314 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-12-12T06:12:37,314 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 in 247 msec 2024-12-12T06:12:37,316 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:37,319 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-12-12T06:12:37,319 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 in 262 msec 2024-12-12T06:12:37,321 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/WALs/ffbfd3107920,33455,1733983951151-splitting dir is empty, no logs to split. 2024-12-12T06:12:37,321 INFO [PEWorker-1 {}] master.SplitWALManager(105): ffbfd3107920,33455,1733983951151 WAL count=0, meta=true 2024-12-12T06:12:37,321 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if ffbfd3107920,33455,1733983951151 WAL splitting is done? wals=0, meta=true 2024-12-12T06:12:37,323 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-12T06:12:37,324 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-12T06:12:37,326 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-12T06:12:37,476 DEBUG [ffbfd3107920:45209 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-12T06:12:37,476 DEBUG [ffbfd3107920:45209 {}] balancer.BalancerClusterState(204): Hosts are {ffbfd3107920=0} racks are {/default-rack=0} 2024-12-12T06:12:37,477 DEBUG [ffbfd3107920:45209 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-12T06:12:37,477 DEBUG [ffbfd3107920:45209 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-12T06:12:37,477 DEBUG [ffbfd3107920:45209 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-12T06:12:37,477 DEBUG [ffbfd3107920:45209 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-12T06:12:37,477 INFO [ffbfd3107920:45209 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-12T06:12:37,477 INFO [ffbfd3107920:45209 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-12T06:12:37,477 DEBUG [ffbfd3107920:45209 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-12T06:12:37,478 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=63 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:37,479 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ffbfd3107920,42275,1733983951224, state=OPENING 2024-12-12T06:12:37,535 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42275-0x10018b511340003, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:37,535 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46199-0x10018b511340001, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:37,535 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45209-0x10018b511340000, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:37,535 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-12T06:12:37,535 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:37,535 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:37,535 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:37,535 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=ffbfd3107920,42275,1733983951224}] 2024-12-12T06:12:37,656 INFO [regionserver/ffbfd3107920:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-12T06:12:37,656 INFO [regionserver/ffbfd3107920:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-12T06:12:37,693 INFO [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=64}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-12T06:12:37,693 INFO [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-12T06:12:37,694 INFO [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=64}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-12T06:12:37,696 INFO [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ffbfd3107920%2C42275%2C1733983951224.meta, suffix=.meta, logDir=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/WALs/ffbfd3107920,42275,1733983951224, archiveDir=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/oldWALs, maxLogs=32 2024-12-12T06:12:37,709 DEBUG [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=64}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/WALs/ffbfd3107920,42275,1733983951224/ffbfd3107920%2C42275%2C1733983951224.meta.1733983957697.meta, exclude list is [], retry=0 2024-12-12T06:12:37,712 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38541,DS-2e86db76-0bd9-4ac5-b9f1-045da95823f2,DISK] 2024-12-12T06:12:37,712 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42285,DS-6cdf379f-c6b3-472d-9e31-f23a652ce941,DISK] 2024-12-12T06:12:37,712 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43493,DS-bf6839c4-19ca-45ad-a024-f2d33b255052,DISK] 2024-12-12T06:12:37,715 INFO [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/WALs/ffbfd3107920,42275,1733983951224/ffbfd3107920%2C42275%2C1733983951224.meta.1733983957697.meta 2024-12-12T06:12:37,715 DEBUG [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39173:39173),(127.0.0.1/127.0.0.1:42655:42655),(127.0.0.1/127.0.0.1:44435:44435)] 2024-12-12T06:12:37,715 DEBUG [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-12T06:12:37,716 DEBUG [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=64}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-12T06:12:37,716 DEBUG [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-12T06:12:37,716 INFO [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-12T06:12:37,716 DEBUG [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-12T06:12:37,716 DEBUG [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-12T06:12:37,716 DEBUG [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-12T06:12:37,716 DEBUG [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-12T06:12:37,718 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-12T06:12:37,719 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-12T06:12:37,719 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-12T06:12:37,719 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-12T06:12:37,720 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-12T06:12:37,721 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-12T06:12:37,721 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-12T06:12:37,721 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-12T06:12:37,721 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-12T06:12:37,723 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-12T06:12:37,723 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-12T06:12:37,723 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-12T06:12:37,724 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-12T06:12:37,725 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-12T06:12:37,725 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-12T06:12:37,725 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-12T06:12:37,726 DEBUG [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-12T06:12:37,726 DEBUG [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/hbase/meta/1588230740 2024-12-12T06:12:37,729 DEBUG [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/hbase/meta/1588230740 2024-12-12T06:12:37,730 INFO [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-12-12T06:12:37,746 DEBUG [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5793): Applied 277, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=64, path=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-12-12T06:12:37,749 INFO [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=29.41 KB heapSize=50.67 KB 2024-12-12T06:12:37,819 DEBUG [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/hbase/meta/1588230740/.tmp/info/7cd5eccda9e140b7a007e15da16ef397 is 138, key is test,bbb,1733983953560.16838c5df0ab4e3938ad84e9a080def9./info:regioninfo/1733983954938/Put/seqid=0 2024-12-12T06:12:37,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42285 is added to blk_1073741866_1042 (size=36381) 2024-12-12T06:12:37,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38541 is added to blk_1073741866_1042 (size=36381) 2024-12-12T06:12:37,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43493 is added to blk_1073741866_1042 (size=36381) 2024-12-12T06:12:37,833 INFO [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.26 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/hbase/meta/1588230740/.tmp/info/7cd5eccda9e140b7a007e15da16ef397 2024-12-12T06:12:37,893 DEBUG [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/hbase/meta/1588230740/.tmp/ns/eee174fd2f5d4ec39a98bfad83e13735 is 43, key is default/ns:d/1733983953349/Put/seqid=0 2024-12-12T06:12:37,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43493 is added to blk_1073741867_1043 (size=5153) 2024-12-12T06:12:37,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42285 is added to blk_1073741867_1043 (size=5153) 2024-12-12T06:12:37,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38541 is added to blk_1073741867_1043 (size=5153) 2024-12-12T06:12:37,902 INFO [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/hbase/meta/1588230740/.tmp/ns/eee174fd2f5d4ec39a98bfad83e13735 2024-12-12T06:12:37,926 DEBUG [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/hbase/meta/1588230740/.tmp/table/71047e159fa243229c4fff7fe4b2a4af is 40, key is test/table:state/1733983954990/Put/seqid=0 2024-12-12T06:12:37,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38541 is added to blk_1073741868_1044 (size=5165) 2024-12-12T06:12:37,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42285 is added to blk_1073741868_1044 (size=5165) 2024-12-12T06:12:37,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43493 is added to blk_1073741868_1044 (size=5165) 2024-12-12T06:12:37,933 INFO [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=72 B at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/hbase/meta/1588230740/.tmp/table/71047e159fa243229c4fff7fe4b2a4af 2024-12-12T06:12:37,941 DEBUG [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/hbase/meta/1588230740/.tmp/info/7cd5eccda9e140b7a007e15da16ef397 as hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/hbase/meta/1588230740/info/7cd5eccda9e140b7a007e15da16ef397 2024-12-12T06:12:37,950 INFO [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/hbase/meta/1588230740/info/7cd5eccda9e140b7a007e15da16ef397, entries=273, sequenceid=64, filesize=35.5 K 2024-12-12T06:12:37,953 DEBUG [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/hbase/meta/1588230740/.tmp/ns/eee174fd2f5d4ec39a98bfad83e13735 as hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/hbase/meta/1588230740/ns/eee174fd2f5d4ec39a98bfad83e13735 2024-12-12T06:12:37,961 INFO [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/hbase/meta/1588230740/ns/eee174fd2f5d4ec39a98bfad83e13735, entries=2, sequenceid=64, filesize=5.0 K 2024-12-12T06:12:37,962 DEBUG [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/hbase/meta/1588230740/.tmp/table/71047e159fa243229c4fff7fe4b2a4af as hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/hbase/meta/1588230740/table/71047e159fa243229c4fff7fe4b2a4af 2024-12-12T06:12:37,971 INFO [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/hbase/meta/1588230740/table/71047e159fa243229c4fff7fe4b2a4af, entries=2, sequenceid=64, filesize=5.0 K 2024-12-12T06:12:37,976 INFO [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(3140): Finished flush of dataSize ~29.41 KB/30111, heapSize ~50.38 KB/51584, currentSize=0 B/0 for 1588230740 in 225ms, sequenceid=64, compaction requested=false; wal=null 2024-12-12T06:12:37,977 DEBUG [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-12T06:12:37,981 DEBUG [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-12-12T06:12:37,982 DEBUG [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-12T06:12:37,982 DEBUG [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-12T06:12:37,983 DEBUG [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-12T06:12:37,986 DEBUG [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-12T06:12:37,988 DEBUG [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/hbase/meta/1588230740/recovered.edits/64.seqid, newMaxSeqId=64, maxSeqId=1 2024-12-12T06:12:37,989 INFO [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=65; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63410934, jitterRate=-0.055103451013565063}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-12T06:12:37,990 DEBUG [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-12T06:12:37,990 DEBUG [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733983957716Writing region info on filesystem at 1733983957716Initializing all the Stores at 1733983957717 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733983957717Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733983957717Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733983957717Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733983957718 (+1 ms)Obtaining lock to block concurrent updates at 1733983957749 (+31 ms)Preparing flush snapshotting stores in 1588230740 at 1733983957749Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=30111, getHeapSize=51824, getOffHeapSize=0, getCellsCount=277 at 1733983957752 (+3 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733983957752Flushing 1588230740/info: creating writer at 1733983957754 (+2 ms)Flushing 1588230740/info: appending metadata at 1733983957811 (+57 ms)Flushing 1588230740/info: closing flushed file at 1733983957813 (+2 ms)Flushing 1588230740/ns: creating writer at 1733983957876 (+63 ms)Flushing 1588230740/ns: appending metadata at 1733983957892 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1733983957892Flushing 1588230740/table: creating writer at 1733983957910 (+18 ms)Flushing 1588230740/table: appending metadata at 1733983957924 (+14 ms)Flushing 1588230740/table: closing flushed file at 1733983957924Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@193eb630: reopening flushed file at 1733983957940 (+16 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@12311c2e: reopening flushed file at 1733983957952 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7bd85583: reopening flushed file at 1733983957961 (+9 ms)Finished flush of dataSize ~29.41 KB/30111, heapSize ~50.38 KB/51584, currentSize=0 B/0 for 1588230740 in 225ms, sequenceid=64, compaction requested=false; wal=null at 1733983957976 (+15 ms)Cleaning up temporary data from old regions at 1733983957982 (+6 ms)Running coprocessor post-open hooks at 1733983957990 (+8 ms)Region opened successfully at 1733983957990 2024-12-12T06:12:37,991 INFO [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=64, masterSystemTime=1733983957688 2024-12-12T06:12:37,993 DEBUG [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-12T06:12:37,994 INFO [RS_OPEN_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_META, pid=64}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-12T06:12:37,994 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=63 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=65, regionLocation=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:37,995 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ffbfd3107920,42275,1733983951224, state=OPEN 2024-12-12T06:12:38,039 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45209-0x10018b511340000, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:38,039 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42275-0x10018b511340003, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:38,039 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46199-0x10018b511340001, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:38,039 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=64, ppid=63, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:38,039 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:38,039 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:38,039 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:38,042 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-12-12T06:12:38,042 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ffbfd3107920,42275,1733983951224 in 504 msec 2024-12-12T06:12:38,045 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=60 2024-12-12T06:12:38,045 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 719 msec 2024-12-12T06:12:38,045 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): ffbfd3107920,33455,1733983951151 had 9 regions 2024-12-12T06:12:38,046 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=60, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true, isMeta: false 2024-12-12T06:12:38,048 INFO [PEWorker-1 {}] master.SplitWALManager(105): ffbfd3107920,33455,1733983951151 WAL count=1, meta=false 2024-12-12T06:12:38,048 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=60, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728}] 2024-12-12T06:12:38,050 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:38,051 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048}] 2024-12-12T06:12:38,205 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=66 2024-12-12T06:12:38,219 INFO [RS_LOG_REPLAY_OPS-regionserver/ffbfd3107920:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(299): Splitting hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/WALs/ffbfd3107920,33455,1733983951151-splitting/ffbfd3107920%2C33455%2C1733983951151.1733983952728, size=2.7 K (2769bytes) 2024-12-12T06:12:38,219 INFO [RS_LOG_REPLAY_OPS-regionserver/ffbfd3107920:0-0 {event_type=RS_LOG_REPLAY, pid=66}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/WALs/ffbfd3107920,33455,1733983951151-splitting/ffbfd3107920%2C33455%2C1733983951151.1733983952728 2024-12-12T06:12:38,220 INFO [RS_LOG_REPLAY_OPS-regionserver/ffbfd3107920:0-0 {event_type=RS_LOG_REPLAY, pid=66}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/WALs/ffbfd3107920,33455,1733983951151-splitting/ffbfd3107920%2C33455%2C1733983951151.1733983952728 after 1ms 2024-12-12T06:12:38,222 INFO [RS_LOG_REPLAY_OPS-regionserver/ffbfd3107920:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(310): Open hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/WALs/ffbfd3107920,33455,1733983951151-splitting/ffbfd3107920%2C33455%2C1733983951151.1733983952728 took 3ms 2024-12-12T06:12:38,225 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ffbfd3107920:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for ae428798a6433b4d48a205ab3d14ba42: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-12T06:12:38,227 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ffbfd3107920:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 60c948f12e02b6ab0066426b6aceea80: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-12T06:12:38,229 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ffbfd3107920:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 5d34bdb8a1414fb25a475fb97fc440b4: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-12T06:12:38,230 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ffbfd3107920:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for d86220c39cb75f305e7ed4d4c9f1191c: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-12T06:12:38,232 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ffbfd3107920:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 145c6ba0291376a96e3a37f6b274f63b: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-12T06:12:38,234 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ffbfd3107920:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 1202ee0cad3e428893fcdbc630d4f9b9: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-12T06:12:38,236 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ffbfd3107920:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 8545c1fe97252abe3447dcfaebd5cfa5: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-12T06:12:38,238 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ffbfd3107920:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 57995ad3e527ada56b846564833341d1: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-12T06:12:38,240 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ffbfd3107920:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 4584d26a35c36968de829b34ac08fb5d: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-12T06:12:38,240 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ffbfd3107920:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/WALs/ffbfd3107920,33455,1733983951151-splitting/ffbfd3107920%2C33455%2C1733983951151.1733983952728 so closing down 2024-12-12T06:12:38,240 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ffbfd3107920:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-12T06:12:38,240 INFO [RS_LOG_REPLAY_OPS-regionserver/ffbfd3107920:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.OutputSink(145): 3 split writer threads finished 2024-12-12T06:12:38,241 INFO [RS_LOG_REPLAY_OPS-regionserver/ffbfd3107920:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(425): Processed 10 edits across 0 Regions in 18 ms; skipped=10; WAL=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/WALs/ffbfd3107920,33455,1733983951151-splitting/ffbfd3107920%2C33455%2C1733983951151.1733983952728, size=2.7 K, length=2769, corrupted=false, cancelled=false 2024-12-12T06:12:38,241 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ffbfd3107920:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(428): Completed split of hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/WALs/ffbfd3107920,33455,1733983951151-splitting/ffbfd3107920%2C33455%2C1733983951151.1733983952728, journal: Splitting hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/WALs/ffbfd3107920,33455,1733983951151-splitting/ffbfd3107920%2C33455%2C1733983951151.1733983952728, size=2.7 K (2769bytes) at 1733983958219Finishing writing output for hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/WALs/ffbfd3107920,33455,1733983951151-splitting/ffbfd3107920%2C33455%2C1733983951151.1733983952728 so closing down at 1733983958240 (+21 ms)3 split writer threads finished at 1733983958241 (+1 ms)Processed 10 edits across 0 Regions in 18 ms; skipped=10; WAL=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/WALs/ffbfd3107920,33455,1733983951151-splitting/ffbfd3107920%2C33455%2C1733983951151.1733983952728, size=2.7 K, length=2769, corrupted=false, cancelled=false at 1733983958241 2024-12-12T06:12:38,241 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ffbfd3107920:0-0 {event_type=RS_LOG_REPLAY, pid=66}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/WALs/ffbfd3107920,33455,1733983951151-splitting/ffbfd3107920%2C33455%2C1733983951151.1733983952728 2024-12-12T06:12:38,241 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ffbfd3107920:0-0 {event_type=RS_LOG_REPLAY, pid=66}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=66 2024-12-12T06:12:38,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45209 {}] master.HMaster(4169): Remote procedure done, pid=66 2024-12-12T06:12:38,246 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/WALs/ffbfd3107920,33455,1733983951151-splitting/ffbfd3107920%2C33455%2C1733983951151.1733983952728 to hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/oldWALs 2024-12-12T06:12:38,249 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=66, resume processing ppid=65 2024-12-12T06:12:38,250 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 in 195 msec 2024-12-12T06:12:38,251 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:38,255 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=60 2024-12-12T06:12:38,255 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 in 204 msec 2024-12-12T06:12:38,257 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/WALs/ffbfd3107920,33455,1733983951151-splitting dir is empty, no logs to split. 2024-12-12T06:12:38,257 INFO [PEWorker-1 {}] master.SplitWALManager(105): ffbfd3107920,33455,1733983951151 WAL count=0, meta=false 2024-12-12T06:12:38,257 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if ffbfd3107920,33455,1733983951151 WAL splitting is done? wals=0, meta=false 2024-12-12T06:12:38,260 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN}, {pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN}, {pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN}, {pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8545c1fe97252abe3447dcfaebd5cfa5, ASSIGN}, {pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1202ee0cad3e428893fcdbc630d4f9b9, ASSIGN}, {pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ae428798a6433b4d48a205ab3d14ba42, ASSIGN}, {pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4584d26a35c36968de829b34ac08fb5d, ASSIGN}, {pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d86220c39cb75f305e7ed4d4c9f1191c, ASSIGN}] 2024-12-12T06:12:38,262 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d86220c39cb75f305e7ed4d4c9f1191c, ASSIGN 2024-12-12T06:12:38,263 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d86220c39cb75f305e7ed4d4c9f1191c, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-12T06:12:38,265 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4584d26a35c36968de829b34ac08fb5d, ASSIGN 2024-12-12T06:12:38,266 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4584d26a35c36968de829b34ac08fb5d, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-12T06:12:38,267 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ae428798a6433b4d48a205ab3d14ba42, ASSIGN 2024-12-12T06:12:38,268 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ae428798a6433b4d48a205ab3d14ba42, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-12T06:12:38,269 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1202ee0cad3e428893fcdbc630d4f9b9, ASSIGN 2024-12-12T06:12:38,269 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=1202ee0cad3e428893fcdbc630d4f9b9, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-12T06:12:38,270 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8545c1fe97252abe3447dcfaebd5cfa5, ASSIGN 2024-12-12T06:12:38,271 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8545c1fe97252abe3447dcfaebd5cfa5, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-12T06:12:38,272 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN 2024-12-12T06:12:38,273 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-12T06:12:38,274 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN 2024-12-12T06:12:38,275 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-12T06:12:38,276 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN 2024-12-12T06:12:38,277 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-12T06:12:38,414 DEBUG [ffbfd3107920:45209 {}] balancer.BalancerClusterState(204): Hosts are {ffbfd3107920=0} racks are {/default-rack=0} 2024-12-12T06:12:38,414 DEBUG [ffbfd3107920:45209 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-12T06:12:38,414 DEBUG [ffbfd3107920:45209 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-12T06:12:38,414 DEBUG [ffbfd3107920:45209 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-12T06:12:38,414 DEBUG [ffbfd3107920:45209 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-12T06:12:38,414 INFO [ffbfd3107920:45209 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-12T06:12:38,414 INFO [ffbfd3107920:45209 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-12T06:12:38,414 DEBUG [ffbfd3107920:45209 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-12T06:12:38,415 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=72 updating hbase:meta row=ae428798a6433b4d48a205ab3d14ba42, regionState=OPENING, regionLocation=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:38,415 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=1202ee0cad3e428893fcdbc630d4f9b9, regionState=OPENING, regionLocation=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:38,416 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=5d34bdb8a1414fb25a475fb97fc440b4, regionState=OPENING, regionLocation=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:38,416 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=73 updating hbase:meta row=4584d26a35c36968de829b34ac08fb5d, regionState=OPENING, regionLocation=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:38,416 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server ffbfd3107920:33455 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ffbfd3107920/172.17.0.2:33455 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-12T06:12:38,416 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=145c6ba0291376a96e3a37f6b274f63b, regionState=OPENING, regionLocation=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:38,417 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=74 updating hbase:meta row=d86220c39cb75f305e7ed4d4c9f1191c, regionState=OPENING, regionLocation=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:38,417 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=57995ad3e527ada56b846564833341d1, regionState=OPENING, regionLocation=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:38,417 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ffbfd3107920,33455,1733983951151, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=ffbfd3107920,33455,1733983951151, seqNum=-1, error=java.net.ConnectException: Call to address=ffbfd3107920:33455 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ffbfd3107920/172.17.0.2:33455 2024-12-12T06:12:38,418 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=8545c1fe97252abe3447dcfaebd5cfa5, regionState=OPENING, regionLocation=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:38,418 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=ffbfd3107920,33455,1733983951151, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-12-12T06:12:38,418 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=ffbfd3107920,33455,1733983951151, seqNum=-1 from cache 2024-12-12T06:12:38,419 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address ffbfd3107920:33455 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ffbfd3107920/172.17.0.2:33455 2024-12-12T06:12:38,419 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server ffbfd3107920:33455 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ffbfd3107920/172.17.0.2:33455 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-12T06:12:38,420 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ffbfd3107920,33455,1733983951151, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=ffbfd3107920:33455 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ffbfd3107920/172.17.0.2:33455 2024-12-12T06:12:38,420 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ffbfd3107920,33455,1733983951151, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=ffbfd3107920:33455 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ffbfd3107920/172.17.0.2:33455 2024-12-12T06:12:38,420 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ffbfd3107920,33455,1733983951151, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=ffbfd3107920:33455 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ffbfd3107920/172.17.0.2:33455 2024-12-12T06:12:38,420 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ffbfd3107920,33455,1733983951151, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=ffbfd3107920:33455 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ffbfd3107920/172.17.0.2:33455 2024-12-12T06:12:38,420 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ffbfd3107920,33455,1733983951151, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=ffbfd3107920:33455 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ffbfd3107920/172.17.0.2:33455 2024-12-12T06:12:38,420 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ffbfd3107920,33455,1733983951151, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=ffbfd3107920:33455 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ffbfd3107920/172.17.0.2:33455 2024-12-12T06:12:38,420 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ffbfd3107920,33455,1733983951151, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=ffbfd3107920:33455 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ffbfd3107920/172.17.0.2:33455 2024-12-12T06:12:38,420 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address ffbfd3107920:33455 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ffbfd3107920/172.17.0.2:33455 2024-12-12T06:12:38,525 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-12T06:12:38,525 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ffbfd3107920,42275,1733983951224, seqNum=-1] 2024-12-12T06:12:38,526 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-12T06:12:38,527 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57849, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-12T06:12:38,531 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ae428798a6433b4d48a205ab3d14ba42, ASSIGN because future has completed 2024-12-12T06:12:38,532 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure ae428798a6433b4d48a205ab3d14ba42, server=ffbfd3107920,42275,1733983951224}] 2024-12-12T06:12:38,533 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN because future has completed 2024-12-12T06:12:38,534 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8545c1fe97252abe3447dcfaebd5cfa5, ASSIGN because future has completed 2024-12-12T06:12:38,536 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=1202ee0cad3e428893fcdbc630d4f9b9, ASSIGN because future has completed 2024-12-12T06:12:38,537 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN because future has completed 2024-12-12T06:12:38,538 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224}] 2024-12-12T06:12:38,538 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4584d26a35c36968de829b34ac08fb5d, ASSIGN because future has completed 2024-12-12T06:12:38,539 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN because future has completed 2024-12-12T06:12:38,540 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d86220c39cb75f305e7ed4d4c9f1191c, ASSIGN because future has completed 2024-12-12T06:12:38,541 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure d86220c39cb75f305e7ed4d4c9f1191c, server=ffbfd3107920,46199,1733983951048}] 2024-12-12T06:12:38,543 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 145c6ba0291376a96e3a37f6b274f63b, server=ffbfd3107920,46199,1733983951048}] 2024-12-12T06:12:38,546 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4584d26a35c36968de829b34ac08fb5d, server=ffbfd3107920,42275,1733983951224}] 2024-12-12T06:12:38,548 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1202ee0cad3e428893fcdbc630d4f9b9, server=ffbfd3107920,42275,1733983951224}] 2024-12-12T06:12:38,551 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8545c1fe97252abe3447dcfaebd5cfa5, server=ffbfd3107920,46199,1733983951048}] 2024-12-12T06:12:38,553 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 57995ad3e527ada56b846564833341d1, server=ffbfd3107920,46199,1733983951048}] 2024-12-12T06:12:38,693 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(132): Open test,lll,1733983953560.5d34bdb8a1414fb25a475fb97fc440b4. 2024-12-12T06:12:38,693 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7752): Opening region: {ENCODED => 5d34bdb8a1414fb25a475fb97fc440b4, NAME => 'test,lll,1733983953560.5d34bdb8a1414fb25a475fb97fc440b4.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-12-12T06:12:38,693 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 5d34bdb8a1414fb25a475fb97fc440b4 2024-12-12T06:12:38,694 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(898): Instantiated test,lll,1733983953560.5d34bdb8a1414fb25a475fb97fc440b4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-12T06:12:38,694 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7794): checking encryption for 5d34bdb8a1414fb25a475fb97fc440b4 2024-12-12T06:12:38,694 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7797): checking classloading for 5d34bdb8a1414fb25a475fb97fc440b4 2024-12-12T06:12:38,696 INFO [StoreOpener-5d34bdb8a1414fb25a475fb97fc440b4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 5d34bdb8a1414fb25a475fb97fc440b4 2024-12-12T06:12:38,697 INFO [StoreOpener-5d34bdb8a1414fb25a475fb97fc440b4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5d34bdb8a1414fb25a475fb97fc440b4 columnFamilyName family 2024-12-12T06:12:38,697 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] handler.AssignRegionHandler(132): Open test,iii,1733983953560.57995ad3e527ada56b846564833341d1. 2024-12-12T06:12:38,698 DEBUG [StoreOpener-5d34bdb8a1414fb25a475fb97fc440b4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-12T06:12:38,698 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7752): Opening region: {ENCODED => 57995ad3e527ada56b846564833341d1, NAME => 'test,iii,1733983953560.57995ad3e527ada56b846564833341d1.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-12-12T06:12:38,698 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 57995ad3e527ada56b846564833341d1 2024-12-12T06:12:38,698 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(898): Instantiated test,iii,1733983953560.57995ad3e527ada56b846564833341d1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-12T06:12:38,698 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7794): checking encryption for 57995ad3e527ada56b846564833341d1 2024-12-12T06:12:38,698 INFO [StoreOpener-5d34bdb8a1414fb25a475fb97fc440b4-1 {}] regionserver.HStore(327): Store=5d34bdb8a1414fb25a475fb97fc440b4/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-12T06:12:38,698 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7797): checking classloading for 57995ad3e527ada56b846564833341d1 2024-12-12T06:12:38,699 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1038): replaying wal for 5d34bdb8a1414fb25a475fb97fc440b4 2024-12-12T06:12:38,699 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/5d34bdb8a1414fb25a475fb97fc440b4 2024-12-12T06:12:38,700 INFO [StoreOpener-57995ad3e527ada56b846564833341d1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 57995ad3e527ada56b846564833341d1 2024-12-12T06:12:38,701 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/5d34bdb8a1414fb25a475fb97fc440b4 2024-12-12T06:12:38,701 INFO [StoreOpener-57995ad3e527ada56b846564833341d1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 57995ad3e527ada56b846564833341d1 columnFamilyName family 2024-12-12T06:12:38,701 DEBUG [StoreOpener-57995ad3e527ada56b846564833341d1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-12T06:12:38,701 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1048): stopping wal replay for 5d34bdb8a1414fb25a475fb97fc440b4 2024-12-12T06:12:38,702 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1060): Cleaning up temporary data for 5d34bdb8a1414fb25a475fb97fc440b4 2024-12-12T06:12:38,702 INFO [StoreOpener-57995ad3e527ada56b846564833341d1-1 {}] regionserver.HStore(327): Store=57995ad3e527ada56b846564833341d1/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-12T06:12:38,702 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1038): replaying wal for 57995ad3e527ada56b846564833341d1 2024-12-12T06:12:38,703 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/57995ad3e527ada56b846564833341d1 2024-12-12T06:12:38,704 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1093): writing seq id for 5d34bdb8a1414fb25a475fb97fc440b4 2024-12-12T06:12:38,704 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/57995ad3e527ada56b846564833341d1 2024-12-12T06:12:38,704 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1048): stopping wal replay for 57995ad3e527ada56b846564833341d1 2024-12-12T06:12:38,704 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1114): Opened 5d34bdb8a1414fb25a475fb97fc440b4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61052942, jitterRate=-0.09024026989936829}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-12T06:12:38,705 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5d34bdb8a1414fb25a475fb97fc440b4 2024-12-12T06:12:38,705 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1060): Cleaning up temporary data for 57995ad3e527ada56b846564833341d1 2024-12-12T06:12:38,705 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1006): Region open journal for 5d34bdb8a1414fb25a475fb97fc440b4: Running coprocessor pre-open hook at 1733983958694Writing region info on filesystem at 1733983958694Initializing all the Stores at 1733983958695 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733983958695Cleaning up temporary data from old regions at 1733983958702 (+7 ms)Running coprocessor post-open hooks at 1733983958705 (+3 ms)Region opened successfully at 1733983958705 2024-12-12T06:12:38,706 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1733983953560.5d34bdb8a1414fb25a475fb97fc440b4., pid=76, masterSystemTime=1733983958685 2024-12-12T06:12:38,707 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1093): writing seq id for 57995ad3e527ada56b846564833341d1 2024-12-12T06:12:38,708 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1733983953560.5d34bdb8a1414fb25a475fb97fc440b4. 2024-12-12T06:12:38,708 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(153): Opened test,lll,1733983953560.5d34bdb8a1414fb25a475fb97fc440b4. 2024-12-12T06:12:38,708 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(132): Open test,fff,1733983953560.4584d26a35c36968de829b34ac08fb5d. 2024-12-12T06:12:38,709 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7752): Opening region: {ENCODED => 4584d26a35c36968de829b34ac08fb5d, NAME => 'test,fff,1733983953560.4584d26a35c36968de829b34ac08fb5d.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-12-12T06:12:38,709 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1114): Opened 57995ad3e527ada56b846564833341d1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70870420, jitterRate=0.056051552295684814}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-12T06:12:38,709 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 57995ad3e527ada56b846564833341d1 2024-12-12T06:12:38,709 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1006): Region open journal for 57995ad3e527ada56b846564833341d1: Running coprocessor pre-open hook at 1733983958699Writing region info on filesystem at 1733983958699Initializing all the Stores at 1733983958700 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733983958700Cleaning up temporary data from old regions at 1733983958705 (+5 ms)Running coprocessor post-open hooks at 1733983958709 (+4 ms)Region opened successfully at 1733983958709 2024-12-12T06:12:38,709 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4584d26a35c36968de829b34ac08fb5d 2024-12-12T06:12:38,709 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(898): Instantiated test,fff,1733983953560.4584d26a35c36968de829b34ac08fb5d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-12T06:12:38,709 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7794): checking encryption for 4584d26a35c36968de829b34ac08fb5d 2024-12-12T06:12:38,709 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7797): checking classloading for 4584d26a35c36968de829b34ac08fb5d 2024-12-12T06:12:38,711 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1733983953560.57995ad3e527ada56b846564833341d1., pid=82, masterSystemTime=1733983958692 2024-12-12T06:12:38,711 WARN [PEWorker-1 {}] procedure2.ProcedureTestingUtility(201): Set Kill before store update in rollback to: true 2024-12-12T06:12:38,712 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:38,712 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1844): CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224 java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-12T06:12:38,714 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegionServer(2266): Finished post open deploy task for test,iii,1733983953560.57995ad3e527ada56b846564833341d1. 2024-12-12T06:12:38,714 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] handler.AssignRegionHandler(153): Opened test,iii,1733983953560.57995ad3e527ada56b846564833341d1. 2024-12-12T06:12:38,714 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(132): Open test,sss,1733983953560.145c6ba0291376a96e3a37f6b274f63b. 2024-12-12T06:12:38,714 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7752): Opening region: {ENCODED => 145c6ba0291376a96e3a37f6b274f63b, NAME => 'test,sss,1733983953560.145c6ba0291376a96e3a37f6b274f63b.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-12-12T06:12:38,714 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 145c6ba0291376a96e3a37f6b274f63b 2024-12-12T06:12:38,714 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(898): Instantiated test,sss,1733983953560.145c6ba0291376a96e3a37f6b274f63b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-12T06:12:38,715 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7794): checking encryption for 145c6ba0291376a96e3a37f6b274f63b 2024-12-12T06:12:38,715 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7797): checking classloading for 145c6ba0291376a96e3a37f6b274f63b 2024-12-12T06:12:38,715 INFO [StoreOpener-4584d26a35c36968de829b34ac08fb5d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4584d26a35c36968de829b34ac08fb5d 2024-12-12T06:12:38,716 INFO [StoreOpener-4584d26a35c36968de829b34ac08fb5d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4584d26a35c36968de829b34ac08fb5d columnFamilyName family 2024-12-12T06:12:38,716 DEBUG [StoreOpener-4584d26a35c36968de829b34ac08fb5d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-12T06:12:38,716 INFO [StoreOpener-145c6ba0291376a96e3a37f6b274f63b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 145c6ba0291376a96e3a37f6b274f63b 2024-12-12T06:12:38,717 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-12T06:12:38,718 INFO [StoreOpener-145c6ba0291376a96e3a37f6b274f63b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 145c6ba0291376a96e3a37f6b274f63b columnFamilyName family 2024-12-12T06:12:38,718 DEBUG [StoreOpener-145c6ba0291376a96e3a37f6b274f63b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-12T06:12:38,721 INFO [StoreOpener-145c6ba0291376a96e3a37f6b274f63b-1 {}] regionserver.HStore(327): Store=145c6ba0291376a96e3a37f6b274f63b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-12T06:12:38,721 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1038): replaying wal for 145c6ba0291376a96e3a37f6b274f63b 2024-12-12T06:12:38,722 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/145c6ba0291376a96e3a37f6b274f63b 2024-12-12T06:12:38,724 INFO [StoreOpener-4584d26a35c36968de829b34ac08fb5d-1 {}] regionserver.HStore(327): Store=4584d26a35c36968de829b34ac08fb5d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-12T06:12:38,724 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1038): replaying wal for 4584d26a35c36968de829b34ac08fb5d 2024-12-12T06:12:38,724 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/145c6ba0291376a96e3a37f6b274f63b 2024-12-12T06:12:38,725 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1048): stopping wal replay for 145c6ba0291376a96e3a37f6b274f63b 2024-12-12T06:12:38,725 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1060): Cleaning up temporary data for 145c6ba0291376a96e3a37f6b274f63b 2024-12-12T06:12:38,725 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/4584d26a35c36968de829b34ac08fb5d 2024-12-12T06:12:38,727 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/4584d26a35c36968de829b34ac08fb5d 2024-12-12T06:12:38,726 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=true, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1845) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 4 more 2024-12-12T06:12:38,727 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-12T06:12:38,727 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1048): stopping wal replay for 4584d26a35c36968de829b34ac08fb5d 2024-12-12T06:12:38,728 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=82, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 57995ad3e527ada56b846564833341d1, server=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:38,728 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-12T06:12:38,728 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1060): Cleaning up temporary data for 4584d26a35c36968de829b34ac08fb5d 2024-12-12T06:12:38,729 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=82, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 57995ad3e527ada56b846564833341d1, server=ffbfd3107920,46199,1733983951048 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-12T06:12:38,729 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1093): writing seq id for 145c6ba0291376a96e3a37f6b274f63b 2024-12-12T06:12:38,730 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1114): Opened 145c6ba0291376a96e3a37f6b274f63b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68068518, jitterRate=0.014299958944320679}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-12T06:12:38,730 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 145c6ba0291376a96e3a37f6b274f63b 2024-12-12T06:12:38,730 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1093): writing seq id for 4584d26a35c36968de829b34ac08fb5d 2024-12-12T06:12:38,730 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1006): Region open journal for 145c6ba0291376a96e3a37f6b274f63b: Running coprocessor pre-open hook at 1733983958715Writing region info on filesystem at 1733983958715Initializing all the Stores at 1733983958716 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733983958716Cleaning up temporary data from old regions at 1733983958725 (+9 ms)Running coprocessor post-open hooks at 1733983958730 (+5 ms)Region opened successfully at 1733983958730 2024-12-12T06:12:38,731 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1733983953560.145c6ba0291376a96e3a37f6b274f63b., pid=78, masterSystemTime=1733983958692 2024-12-12T06:12:38,731 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1114): Opened 4584d26a35c36968de829b34ac08fb5d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67885940, jitterRate=0.011579334735870361}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-12T06:12:38,732 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4584d26a35c36968de829b34ac08fb5d 2024-12-12T06:12:38,732 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1006): Region open journal for 4584d26a35c36968de829b34ac08fb5d: Running coprocessor pre-open hook at 1733983958709Writing region info on filesystem at 1733983958709Initializing all the Stores at 1733983958711 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733983958711Cleaning up temporary data from old regions at 1733983958728 (+17 ms)Running coprocessor post-open hooks at 1733983958732 (+4 ms)Region opened successfully at 1733983958732 2024-12-12T06:12:38,733 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1733983953560.4584d26a35c36968de829b34ac08fb5d., pid=79, masterSystemTime=1733983958685 2024-12-12T06:12:38,734 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1733983953560.145c6ba0291376a96e3a37f6b274f63b. 2024-12-12T06:12:38,734 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(153): Opened test,sss,1733983953560.145c6ba0291376a96e3a37f6b274f63b. 2024-12-12T06:12:38,734 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(132): Open test,,1733983953560.8545c1fe97252abe3447dcfaebd5cfa5. 2024-12-12T06:12:38,735 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7752): Opening region: {ENCODED => 8545c1fe97252abe3447dcfaebd5cfa5, NAME => 'test,,1733983953560.8545c1fe97252abe3447dcfaebd5cfa5.', STARTKEY => '', ENDKEY => 'bbb'} 2024-12-12T06:12:38,735 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8545c1fe97252abe3447dcfaebd5cfa5 2024-12-12T06:12:38,735 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1733983953560.4584d26a35c36968de829b34ac08fb5d. 2024-12-12T06:12:38,735 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(153): Opened test,fff,1733983953560.4584d26a35c36968de829b34ac08fb5d. 2024-12-12T06:12:38,736 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(132): Open test,ppp,1733983953560.1202ee0cad3e428893fcdbc630d4f9b9. 2024-12-12T06:12:38,736 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7752): Opening region: {ENCODED => 1202ee0cad3e428893fcdbc630d4f9b9, NAME => 'test,ppp,1733983953560.1202ee0cad3e428893fcdbc630d4f9b9.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-12-12T06:12:38,736 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 1202ee0cad3e428893fcdbc630d4f9b9 2024-12-12T06:12:38,736 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(898): Instantiated test,ppp,1733983953560.1202ee0cad3e428893fcdbc630d4f9b9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-12T06:12:38,736 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7794): checking encryption for 1202ee0cad3e428893fcdbc630d4f9b9 2024-12-12T06:12:38,736 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7797): checking classloading for 1202ee0cad3e428893fcdbc630d4f9b9 2024-12-12T06:12:38,735 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(898): Instantiated test,,1733983953560.8545c1fe97252abe3447dcfaebd5cfa5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-12T06:12:38,737 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7794): checking encryption for 8545c1fe97252abe3447dcfaebd5cfa5 2024-12-12T06:12:38,737 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7797): checking classloading for 8545c1fe97252abe3447dcfaebd5cfa5 2024-12-12T06:12:38,738 INFO [StoreOpener-8545c1fe97252abe3447dcfaebd5cfa5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8545c1fe97252abe3447dcfaebd5cfa5 2024-12-12T06:12:38,738 INFO [StoreOpener-1202ee0cad3e428893fcdbc630d4f9b9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 1202ee0cad3e428893fcdbc630d4f9b9 2024-12-12T06:12:38,739 INFO [StoreOpener-8545c1fe97252abe3447dcfaebd5cfa5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8545c1fe97252abe3447dcfaebd5cfa5 columnFamilyName family 2024-12-12T06:12:38,739 DEBUG [StoreOpener-8545c1fe97252abe3447dcfaebd5cfa5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-12T06:12:38,739 INFO [StoreOpener-1202ee0cad3e428893fcdbc630d4f9b9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1202ee0cad3e428893fcdbc630d4f9b9 columnFamilyName family 2024-12-12T06:12:38,739 DEBUG [StoreOpener-1202ee0cad3e428893fcdbc630d4f9b9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-12T06:12:38,740 INFO [StoreOpener-8545c1fe97252abe3447dcfaebd5cfa5-1 {}] regionserver.HStore(327): Store=8545c1fe97252abe3447dcfaebd5cfa5/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-12T06:12:38,740 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1038): replaying wal for 8545c1fe97252abe3447dcfaebd5cfa5 2024-12-12T06:12:38,740 INFO [StoreOpener-1202ee0cad3e428893fcdbc630d4f9b9-1 {}] regionserver.HStore(327): Store=1202ee0cad3e428893fcdbc630d4f9b9/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-12T06:12:38,740 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1038): replaying wal for 1202ee0cad3e428893fcdbc630d4f9b9 2024-12-12T06:12:38,741 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/8545c1fe97252abe3447dcfaebd5cfa5 2024-12-12T06:12:38,741 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-12T06:12:38,741 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-12T06:12:38,741 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-12T06:12:38,741 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/1202ee0cad3e428893fcdbc630d4f9b9 2024-12-12T06:12:38,742 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-12T06:12:38,742 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-12T06:12:38,742 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/8545c1fe97252abe3447dcfaebd5cfa5 2024-12-12T06:12:38,742 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-12T06:12:38,742 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-12T06:12:38,743 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-12T06:12:38,743 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1048): stopping wal replay for 8545c1fe97252abe3447dcfaebd5cfa5 2024-12-12T06:12:38,743 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1060): Cleaning up temporary data for 8545c1fe97252abe3447dcfaebd5cfa5 2024-12-12T06:12:38,743 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-12T06:12:38,744 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/1202ee0cad3e428893fcdbc630d4f9b9 2024-12-12T06:12:38,745 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1048): stopping wal replay for 1202ee0cad3e428893fcdbc630d4f9b9 2024-12-12T06:12:38,745 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1060): Cleaning up temporary data for 1202ee0cad3e428893fcdbc630d4f9b9 2024-12-12T06:12:38,746 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1093): writing seq id for 8545c1fe97252abe3447dcfaebd5cfa5 2024-12-12T06:12:38,747 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1114): Opened 8545c1fe97252abe3447dcfaebd5cfa5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66969789, jitterRate=-0.002072378993034363}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-12T06:12:38,748 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1093): writing seq id for 1202ee0cad3e428893fcdbc630d4f9b9 2024-12-12T06:12:38,748 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8545c1fe97252abe3447dcfaebd5cfa5 2024-12-12T06:12:38,748 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1006): Region open journal for 8545c1fe97252abe3447dcfaebd5cfa5: Running coprocessor pre-open hook at 1733983958737Writing region info on filesystem at 1733983958737Initializing all the Stores at 1733983958737Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733983958737Cleaning up temporary data from old regions at 1733983958743 (+6 ms)Running coprocessor post-open hooks at 1733983958748 (+5 ms)Region opened successfully at 1733983958748 2024-12-12T06:12:38,749 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1733983953560.8545c1fe97252abe3447dcfaebd5cfa5., pid=81, masterSystemTime=1733983958692 2024-12-12T06:12:38,749 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1114): Opened 1202ee0cad3e428893fcdbc630d4f9b9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63972810, jitterRate=-0.04673084616661072}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-12T06:12:38,749 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1202ee0cad3e428893fcdbc630d4f9b9 2024-12-12T06:12:38,749 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1006): Region open journal for 1202ee0cad3e428893fcdbc630d4f9b9: Running coprocessor pre-open hook at 1733983958736Writing region info on filesystem at 1733983958736Initializing all the Stores at 1733983958737 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733983958738 (+1 ms)Cleaning up temporary data from old regions at 1733983958745 (+7 ms)Running coprocessor post-open hooks at 1733983958749 (+4 ms)Region opened successfully at 1733983958749 2024-12-12T06:12:38,750 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45209 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-12T06:12:38,751 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-12T06:12:38,752 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-12T06:12:38,752 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-12T06:12:38,752 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:38,752 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-12T06:12:38,752 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:38,752 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=82, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-12T06:12:38,752 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:38,752 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-12T06:12:38,752 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:38,752 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1733983953560.1202ee0cad3e428893fcdbc630d4f9b9., pid=80, masterSystemTime=1733983958685 2024-12-12T06:12:38,752 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=81, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-12T06:12:38,752 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:38,753 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-12T06:12:38,753 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:38,753 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-12T06:12:38,753 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:38,753 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-12T06:12:38,753 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:38,753 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-12T06:12:38,753 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-12T06:12:38,753 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-12T06:12:38,753 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:38,753 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-12T06:12:38,753 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45209 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-12T06:12:38,754 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-12T06:12:38,755 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-12T06:12:38,755 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=60c948f12e02b6ab0066426b6aceea80, REOPEN/MOVE 2024-12-12T06:12:38,759 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true 2024-12-12T06:12:38,759 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:38,759 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:38,759 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-12T06:12:38,759 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:38,759 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:38,759 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:38,759 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN 2024-12-12T06:12:38,759 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN 2024-12-12T06:12:38,759 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN 2024-12-12T06:12:38,759 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8545c1fe97252abe3447dcfaebd5cfa5, ASSIGN 2024-12-12T06:12:38,759 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1202ee0cad3e428893fcdbc630d4f9b9, ASSIGN 2024-12-12T06:12:38,759 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae428798a6433b4d48a205ab3d14ba42, ASSIGN 2024-12-12T06:12:38,759 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4584d26a35c36968de829b34ac08fb5d, ASSIGN 2024-12-12T06:12:38,759 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d86220c39cb75f305e7ed4d4c9f1191c, ASSIGN 2024-12-12T06:12:38,759 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure ae428798a6433b4d48a205ab3d14ba42, server=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:38,760 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:38,760 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure d86220c39cb75f305e7ed4d4c9f1191c, server=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:38,760 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 145c6ba0291376a96e3a37f6b274f63b, server=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:38,760 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4584d26a35c36968de829b34ac08fb5d, server=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:38,760 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1202ee0cad3e428893fcdbc630d4f9b9, server=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:38,760 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=81, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8545c1fe97252abe3447dcfaebd5cfa5, server=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:38,760 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=82, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 57995ad3e527ada56b846564833341d1, server=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:38,760 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:38,760 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:38,760 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:38,760 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:38,760 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:38,760 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:38,760 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:38,760 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:38,760 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN 2024-12-12T06:12:38,760 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:38,760 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN 2024-12-12T06:12:38,760 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:38,760 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN 2024-12-12T06:12:38,760 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8545c1fe97252abe3447dcfaebd5cfa5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:38,760 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8545c1fe97252abe3447dcfaebd5cfa5, ASSIGN 2024-12-12T06:12:38,760 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1202ee0cad3e428893fcdbc630d4f9b9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:38,760 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1202ee0cad3e428893fcdbc630d4f9b9, ASSIGN 2024-12-12T06:12:38,760 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae428798a6433b4d48a205ab3d14ba42, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:38,760 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae428798a6433b4d48a205ab3d14ba42, ASSIGN 2024-12-12T06:12:38,760 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4584d26a35c36968de829b34ac08fb5d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:38,760 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4584d26a35c36968de829b34ac08fb5d, ASSIGN 2024-12-12T06:12:38,760 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d86220c39cb75f305e7ed4d4c9f1191c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:38,760 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d86220c39cb75f305e7ed4d4c9f1191c, ASSIGN 2024-12-12T06:12:38,760 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure ae428798a6433b4d48a205ab3d14ba42, server=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:38,760 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:38,760 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure d86220c39cb75f305e7ed4d4c9f1191c, server=ffbfd3107920,46199,1733983951048 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:38,761 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 145c6ba0291376a96e3a37f6b274f63b, server=ffbfd3107920,46199,1733983951048 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:38,761 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4584d26a35c36968de829b34ac08fb5d, server=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:38,761 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1202ee0cad3e428893fcdbc630d4f9b9, server=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:38,761 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=81, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8545c1fe97252abe3447dcfaebd5cfa5, server=ffbfd3107920,46199,1733983951048 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:38,761 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=82, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 57995ad3e527ada56b846564833341d1, server=ffbfd3107920,46199,1733983951048 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:38,761 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 17 msec 2024-12-12T06:12:38,763 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=65 2024-12-12T06:12:38,763 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ffbfd3107920,42275,1733983951224, table=hbase:meta, region=1588230740 2024-12-12T06:12:38,763 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ffbfd3107920,42275,1733983951224, state=OPEN 2024-12-12T06:12:38,775 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "ffbfd3107920" port: 42275 start_code: 1733983951224 } transition { transition_code: OPENED region_info { region_id: 1733983953560 table_name { namespace: "default" qualifier: "test" } start_key: "ppp" end_key: "qqq" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 80 initiating_master_active_time: 1733983951346 }; retry (#0) after 1004ms delay (Master is coming online...). org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-12-12T06:12:38,775 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "ffbfd3107920" port: 46199 start_code: 1733983951048 } transition { transition_code: OPENED region_info { region_id: 1733983953560 table_name { namespace: "default" qualifier: "test" } start_key: "" end_key: "bbb" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 81 initiating_master_active_time: 1733983951346 }; retry (#0) after 1002ms delay (Master is coming online...). org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-12-12T06:12:38,779 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test' 2024-12-12T06:12:38,814 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45209 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-12T06:12:38,816 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45209 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-12T06:12:38,831 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42275-0x10018b511340003, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:38,831 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46199-0x10018b511340001, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:38,831 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45209-0x10018b511340000, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:38,831 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:38,831 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:38,831 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:38,832 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN to state=OFFLINE, location=null, table=test, region=145c6ba0291376a96e3a37f6b274f63b to restore RIT 2024-12-12T06:12:38,832 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN to state=OFFLINE, location=null, table=test, region=57995ad3e527ada56b846564833341d1 to restore RIT 2024-12-12T06:12:38,832 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN to state=OFFLINE, location=null, table=test, region=5d34bdb8a1414fb25a475fb97fc440b4 to restore RIT 2024-12-12T06:12:38,833 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8545c1fe97252abe3447dcfaebd5cfa5, ASSIGN to state=OFFLINE, location=null, table=test, region=8545c1fe97252abe3447dcfaebd5cfa5 to restore RIT 2024-12-12T06:12:38,833 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1202ee0cad3e428893fcdbc630d4f9b9, ASSIGN to state=OFFLINE, location=null, table=test, region=1202ee0cad3e428893fcdbc630d4f9b9 to restore RIT 2024-12-12T06:12:38,833 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae428798a6433b4d48a205ab3d14ba42, ASSIGN to state=OFFLINE, location=null, table=test, region=ae428798a6433b4d48a205ab3d14ba42 to restore RIT 2024-12-12T06:12:38,833 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4584d26a35c36968de829b34ac08fb5d, ASSIGN to state=OFFLINE, location=null, table=test, region=4584d26a35c36968de829b34ac08fb5d to restore RIT 2024-12-12T06:12:38,833 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d86220c39cb75f305e7ed4d4c9f1191c, ASSIGN to state=OFFLINE, location=null, table=test, region=d86220c39cb75f305e7ed4d4c9f1191c to restore RIT 2024-12-12T06:12:38,833 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733983988833 2024-12-12T06:12:38,833 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-12T06:12:38,833 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-12T06:12:38,835 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-12T06:12:38,837 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-12T06:12:38,837 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=81, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8545c1fe97252abe3447dcfaebd5cfa5, server=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:38,837 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-12T06:12:38,837 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=81, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8545c1fe97252abe3447dcfaebd5cfa5, server=ffbfd3107920,46199,1733983951048 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-12T06:12:38,864 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8545c1fe97252abe3447dcfaebd5cfa5, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:38,865 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16838c5df0ab4e3938ad84e9a080def9, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:38,865 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2eeaafb9a1964f4c6899fa5dde05a463, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:38,865 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c98bb859cc9e047b211ddb4c5b967ce5, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:38,866 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b935ee7eeea02b161c0ebc0d05527e91, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:38,866 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4584d26a35c36968de829b34ac08fb5d, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:38,866 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b717f65da42dacb67935407f094e8467, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:38,867 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f2b6a6b11f14670aca9dccceb20d9630, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:38,867 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57995ad3e527ada56b846564833341d1, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:38,867 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c77db308a7c1c57fe9ef945e17e28127, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:38,867 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=782c52424de7d71cb43b8fbfe8e89f7f, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:38,868 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d34bdb8a1414fb25a475fb97fc440b4, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:38,868 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d86220c39cb75f305e7ed4d4c9f1191c, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:38,868 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be4cb5da32eefc543e09e80b426b6938, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:38,869 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4bf848f8297d89de7b203fcc6678eb5, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:38,869 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1202ee0cad3e428893fcdbc630d4f9b9, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:38,869 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=653572572648db72daea94dd65cfb7f8, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:38,869 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5fe5dc5eb9193cba5339ee508dff867, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:38,870 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=145c6ba0291376a96e3a37f6b274f63b, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:38,870 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5162d3c0b3255050cbeed27e4456e5c2, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:38,870 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d3ce5f64ef1889a3856db9d8c0f63a72, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:38,870 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60c948f12e02b6ab0066426b6aceea80, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=5 2024-12-12T06:12:38,871 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=764fc3b5431e53dfb445323d0bfd39a6, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:38,871 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=269d56dd3a6671f5c9a30401b73ade89, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:38,871 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae428798a6433b4d48a205ab3d14ba42, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:38,871 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=269eed2924060312949b51c10150f409, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:38,871 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-12T06:12:38,871 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733984018871 2024-12-12T06:12:38,872 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733984078872 2024-12-12T06:12:38,872 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 38 msec 2024-12-12T06:12:38,872 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-12T06:12:38,872 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-12T06:12:38,872 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-12T06:12:38,872 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-12T06:12:38,872 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-12T06:12:38,873 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-12T06:12:38,873 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-12T06:12:38,873 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-12T06:12:38,874 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-12T06:12:38,880 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-12T06:12:38,880 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-12T06:12:38,880 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-12T06:12:38,880 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:38,881 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-12T06:12:38,881 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:38,881 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:38,881 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-12T06:12:38,881 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:38,881 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=81, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-12T06:12:38,881 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:38,881 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-12T06:12:38,881 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:38,881 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-12T06:12:38,881 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:38,881 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-12T06:12:38,881 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:38,881 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-12T06:12:38,881 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-12T06:12:38,881 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-12T06:12:38,881 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:38,881 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-12T06:12:38,882 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-12T06:12:38,883 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-12T06:12:38,884 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=60c948f12e02b6ab0066426b6aceea80, REOPEN/MOVE 2024-12-12T06:12:38,887 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true 2024-12-12T06:12:38,887 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:38,887 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:38,888 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-12T06:12:38,888 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:38,888 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:38,888 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:38,888 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN 2024-12-12T06:12:38,888 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN 2024-12-12T06:12:38,888 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN 2024-12-12T06:12:38,888 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8545c1fe97252abe3447dcfaebd5cfa5, ASSIGN 2024-12-12T06:12:38,888 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1202ee0cad3e428893fcdbc630d4f9b9, ASSIGN 2024-12-12T06:12:38,888 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae428798a6433b4d48a205ab3d14ba42, ASSIGN 2024-12-12T06:12:38,888 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4584d26a35c36968de829b34ac08fb5d, ASSIGN 2024-12-12T06:12:38,888 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d86220c39cb75f305e7ed4d4c9f1191c, ASSIGN 2024-12-12T06:12:38,888 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure ae428798a6433b4d48a205ab3d14ba42, server=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:38,888 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:38,888 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure d86220c39cb75f305e7ed4d4c9f1191c, server=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:38,888 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 145c6ba0291376a96e3a37f6b274f63b, server=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:38,888 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4584d26a35c36968de829b34ac08fb5d, server=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:38,888 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1202ee0cad3e428893fcdbc630d4f9b9, server=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:38,888 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=81, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8545c1fe97252abe3447dcfaebd5cfa5, server=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:38,888 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:38,888 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:38,888 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:38,889 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:38,889 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:38,889 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:38,889 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:38,889 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:38,889 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN 2024-12-12T06:12:38,889 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:38,889 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN 2024-12-12T06:12:38,889 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:38,889 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN 2024-12-12T06:12:38,889 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8545c1fe97252abe3447dcfaebd5cfa5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:38,889 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8545c1fe97252abe3447dcfaebd5cfa5, ASSIGN 2024-12-12T06:12:38,889 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1202ee0cad3e428893fcdbc630d4f9b9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:38,889 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1202ee0cad3e428893fcdbc630d4f9b9, ASSIGN 2024-12-12T06:12:38,889 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae428798a6433b4d48a205ab3d14ba42, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:38,889 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae428798a6433b4d48a205ab3d14ba42, ASSIGN 2024-12-12T06:12:38,889 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4584d26a35c36968de829b34ac08fb5d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:38,889 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4584d26a35c36968de829b34ac08fb5d, ASSIGN 2024-12-12T06:12:38,889 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d86220c39cb75f305e7ed4d4c9f1191c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:38,889 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d86220c39cb75f305e7ed4d4c9f1191c, ASSIGN 2024-12-12T06:12:38,889 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure ae428798a6433b4d48a205ab3d14ba42, server=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:38,889 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:38,889 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure d86220c39cb75f305e7ed4d4c9f1191c, server=ffbfd3107920,46199,1733983951048 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:38,890 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 145c6ba0291376a96e3a37f6b274f63b, server=ffbfd3107920,46199,1733983951048 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:38,890 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4584d26a35c36968de829b34ac08fb5d, server=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:38,890 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1202ee0cad3e428893fcdbc630d4f9b9, server=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:38,890 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=81, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8545c1fe97252abe3447dcfaebd5cfa5, server=ffbfd3107920,46199,1733983951048 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:38,890 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 15 msec 2024-12-12T06:12:38,891 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=65 2024-12-12T06:12:38,891 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ffbfd3107920,42275,1733983951224, table=hbase:meta, region=1588230740 2024-12-12T06:12:38,892 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ffbfd3107920,42275,1733983951224, state=OPEN 2024-12-12T06:12:38,909 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46199-0x10018b511340001, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:38,909 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42275-0x10018b511340003, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:38,909 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45209-0x10018b511340000, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:38,909 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:38,909 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:38,909 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:38,910 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN to state=OFFLINE, location=null, table=test, region=145c6ba0291376a96e3a37f6b274f63b to restore RIT 2024-12-12T06:12:38,910 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN to state=OFFLINE, location=null, table=test, region=57995ad3e527ada56b846564833341d1 to restore RIT 2024-12-12T06:12:38,910 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN to state=OFFLINE, location=null, table=test, region=5d34bdb8a1414fb25a475fb97fc440b4 to restore RIT 2024-12-12T06:12:38,910 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8545c1fe97252abe3447dcfaebd5cfa5, ASSIGN to state=OFFLINE, location=null, table=test, region=8545c1fe97252abe3447dcfaebd5cfa5 to restore RIT 2024-12-12T06:12:38,910 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1202ee0cad3e428893fcdbc630d4f9b9, ASSIGN to state=OFFLINE, location=null, table=test, region=1202ee0cad3e428893fcdbc630d4f9b9 to restore RIT 2024-12-12T06:12:38,910 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae428798a6433b4d48a205ab3d14ba42, ASSIGN to state=OFFLINE, location=null, table=test, region=ae428798a6433b4d48a205ab3d14ba42 to restore RIT 2024-12-12T06:12:38,910 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4584d26a35c36968de829b34ac08fb5d, ASSIGN to state=OFFLINE, location=null, table=test, region=4584d26a35c36968de829b34ac08fb5d to restore RIT 2024-12-12T06:12:38,910 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d86220c39cb75f305e7ed4d4c9f1191c, ASSIGN to state=OFFLINE, location=null, table=test, region=d86220c39cb75f305e7ed4d4c9f1191c to restore RIT 2024-12-12T06:12:38,911 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733983988911 2024-12-12T06:12:38,911 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-12T06:12:38,911 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-12T06:12:38,912 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-12T06:12:38,913 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-12T06:12:38,913 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=80, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1202ee0cad3e428893fcdbc630d4f9b9, server=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:38,913 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-12T06:12:38,913 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=80, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1202ee0cad3e428893fcdbc630d4f9b9, server=ffbfd3107920,42275,1733983951224 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-12T06:12:38,918 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8545c1fe97252abe3447dcfaebd5cfa5, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:38,918 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16838c5df0ab4e3938ad84e9a080def9, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:38,919 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2eeaafb9a1964f4c6899fa5dde05a463, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:38,919 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c98bb859cc9e047b211ddb4c5b967ce5, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:38,919 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b935ee7eeea02b161c0ebc0d05527e91, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:38,919 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4584d26a35c36968de829b34ac08fb5d, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:38,919 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b717f65da42dacb67935407f094e8467, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:38,919 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f2b6a6b11f14670aca9dccceb20d9630, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:38,919 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57995ad3e527ada56b846564833341d1, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:38,919 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c77db308a7c1c57fe9ef945e17e28127, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:38,920 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=782c52424de7d71cb43b8fbfe8e89f7f, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:38,920 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d34bdb8a1414fb25a475fb97fc440b4, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:38,920 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d86220c39cb75f305e7ed4d4c9f1191c, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:38,920 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be4cb5da32eefc543e09e80b426b6938, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:38,920 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4bf848f8297d89de7b203fcc6678eb5, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:38,920 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1202ee0cad3e428893fcdbc630d4f9b9, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:38,920 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=653572572648db72daea94dd65cfb7f8, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:38,921 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5fe5dc5eb9193cba5339ee508dff867, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:38,921 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=145c6ba0291376a96e3a37f6b274f63b, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:38,921 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5162d3c0b3255050cbeed27e4456e5c2, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:38,921 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d3ce5f64ef1889a3856db9d8c0f63a72, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:38,921 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60c948f12e02b6ab0066426b6aceea80, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=5 2024-12-12T06:12:38,921 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=764fc3b5431e53dfb445323d0bfd39a6, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:38,921 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=269d56dd3a6671f5c9a30401b73ade89, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:38,922 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae428798a6433b4d48a205ab3d14ba42, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:38,922 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=269eed2924060312949b51c10150f409, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:38,922 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-12T06:12:38,922 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733984018922 2024-12-12T06:12:38,922 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733984078922 2024-12-12T06:12:38,922 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 11 msec 2024-12-12T06:12:38,922 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-12T06:12:38,922 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-12T06:12:38,922 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-12T06:12:38,922 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-12T06:12:38,922 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-12T06:12:38,922 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-12T06:12:38,922 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-12T06:12:38,923 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-12T06:12:38,923 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-12T06:12:38,926 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-12T06:12:38,926 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-12T06:12:38,926 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-12T06:12:38,926 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:38,926 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-12T06:12:38,926 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:38,926 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:38,926 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-12T06:12:38,926 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:38,926 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:38,926 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-12T06:12:38,927 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:38,927 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-12T06:12:38,927 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:38,927 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-12T06:12:38,927 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:38,927 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-12T06:12:38,927 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-12T06:12:38,927 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-12T06:12:38,927 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:38,927 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-12T06:12:38,927 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-12T06:12:38,928 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-12T06:12:38,928 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=60c948f12e02b6ab0066426b6aceea80, REOPEN/MOVE 2024-12-12T06:12:38,930 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true 2024-12-12T06:12:38,930 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:38,930 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:38,930 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-12T06:12:38,930 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:38,930 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:38,930 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:38,930 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN 2024-12-12T06:12:38,930 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN 2024-12-12T06:12:38,930 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN 2024-12-12T06:12:38,930 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8545c1fe97252abe3447dcfaebd5cfa5, ASSIGN 2024-12-12T06:12:38,930 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1202ee0cad3e428893fcdbc630d4f9b9, ASSIGN 2024-12-12T06:12:38,930 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae428798a6433b4d48a205ab3d14ba42, ASSIGN 2024-12-12T06:12:38,930 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4584d26a35c36968de829b34ac08fb5d, ASSIGN 2024-12-12T06:12:38,930 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d86220c39cb75f305e7ed4d4c9f1191c, ASSIGN 2024-12-12T06:12:38,930 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure ae428798a6433b4d48a205ab3d14ba42, server=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:38,930 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:38,930 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure d86220c39cb75f305e7ed4d4c9f1191c, server=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:38,930 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 145c6ba0291376a96e3a37f6b274f63b, server=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:38,930 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4584d26a35c36968de829b34ac08fb5d, server=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:38,930 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1202ee0cad3e428893fcdbc630d4f9b9, server=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:38,930 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:38,930 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:38,930 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:38,931 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:38,931 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:38,931 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:38,931 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:38,931 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:38,931 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN 2024-12-12T06:12:38,931 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:38,931 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN 2024-12-12T06:12:38,931 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:38,931 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN 2024-12-12T06:12:38,931 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8545c1fe97252abe3447dcfaebd5cfa5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:38,931 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8545c1fe97252abe3447dcfaebd5cfa5, ASSIGN 2024-12-12T06:12:38,931 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1202ee0cad3e428893fcdbc630d4f9b9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:38,931 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1202ee0cad3e428893fcdbc630d4f9b9, ASSIGN 2024-12-12T06:12:38,931 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae428798a6433b4d48a205ab3d14ba42, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:38,931 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae428798a6433b4d48a205ab3d14ba42, ASSIGN 2024-12-12T06:12:38,931 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4584d26a35c36968de829b34ac08fb5d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:38,931 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4584d26a35c36968de829b34ac08fb5d, ASSIGN 2024-12-12T06:12:38,931 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d86220c39cb75f305e7ed4d4c9f1191c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:38,931 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d86220c39cb75f305e7ed4d4c9f1191c, ASSIGN 2024-12-12T06:12:38,931 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure ae428798a6433b4d48a205ab3d14ba42, server=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:38,931 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:38,931 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure d86220c39cb75f305e7ed4d4c9f1191c, server=ffbfd3107920,46199,1733983951048 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:38,931 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 145c6ba0291376a96e3a37f6b274f63b, server=ffbfd3107920,46199,1733983951048 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:38,931 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4584d26a35c36968de829b34ac08fb5d, server=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:38,931 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1202ee0cad3e428893fcdbc630d4f9b9, server=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:38,931 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-12-12T06:12:38,932 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=65 2024-12-12T06:12:38,932 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ffbfd3107920,42275,1733983951224, table=hbase:meta, region=1588230740 2024-12-12T06:12:38,933 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ffbfd3107920,42275,1733983951224, state=OPEN 2024-12-12T06:12:38,933 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45209 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-12T06:12:38,933 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45209 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-12T06:12:38,943 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46199-0x10018b511340001, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:38,943 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42275-0x10018b511340003, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:38,943 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45209-0x10018b511340000, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:38,943 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:38,943 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:38,943 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:38,943 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN to state=OFFLINE, location=null, table=test, region=145c6ba0291376a96e3a37f6b274f63b to restore RIT 2024-12-12T06:12:38,944 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN to state=OFFLINE, location=null, table=test, region=57995ad3e527ada56b846564833341d1 to restore RIT 2024-12-12T06:12:38,944 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN to state=OFFLINE, location=null, table=test, region=5d34bdb8a1414fb25a475fb97fc440b4 to restore RIT 2024-12-12T06:12:38,944 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8545c1fe97252abe3447dcfaebd5cfa5, ASSIGN to state=OFFLINE, location=null, table=test, region=8545c1fe97252abe3447dcfaebd5cfa5 to restore RIT 2024-12-12T06:12:38,944 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1202ee0cad3e428893fcdbc630d4f9b9, ASSIGN to state=OFFLINE, location=null, table=test, region=1202ee0cad3e428893fcdbc630d4f9b9 to restore RIT 2024-12-12T06:12:38,944 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae428798a6433b4d48a205ab3d14ba42, ASSIGN to state=OFFLINE, location=null, table=test, region=ae428798a6433b4d48a205ab3d14ba42 to restore RIT 2024-12-12T06:12:38,944 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4584d26a35c36968de829b34ac08fb5d, ASSIGN to state=OFFLINE, location=null, table=test, region=4584d26a35c36968de829b34ac08fb5d to restore RIT 2024-12-12T06:12:38,944 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d86220c39cb75f305e7ed4d4c9f1191c, ASSIGN to state=OFFLINE, location=null, table=test, region=d86220c39cb75f305e7ed4d4c9f1191c to restore RIT 2024-12-12T06:12:38,944 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733983988944 2024-12-12T06:12:38,944 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-12T06:12:38,944 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-12T06:12:38,945 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-12T06:12:38,946 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-12T06:12:38,946 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4584d26a35c36968de829b34ac08fb5d, server=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:38,946 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-12T06:12:38,947 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4584d26a35c36968de829b34ac08fb5d, server=ffbfd3107920,42275,1733983951224 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-12T06:12:38,950 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8545c1fe97252abe3447dcfaebd5cfa5, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:38,950 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16838c5df0ab4e3938ad84e9a080def9, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:38,950 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2eeaafb9a1964f4c6899fa5dde05a463, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:38,950 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c98bb859cc9e047b211ddb4c5b967ce5, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:38,950 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b935ee7eeea02b161c0ebc0d05527e91, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:38,951 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4584d26a35c36968de829b34ac08fb5d, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:38,951 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b717f65da42dacb67935407f094e8467, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:38,951 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f2b6a6b11f14670aca9dccceb20d9630, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:38,951 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57995ad3e527ada56b846564833341d1, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:38,951 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c77db308a7c1c57fe9ef945e17e28127, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:38,951 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=782c52424de7d71cb43b8fbfe8e89f7f, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:38,951 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d34bdb8a1414fb25a475fb97fc440b4, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:38,952 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d86220c39cb75f305e7ed4d4c9f1191c, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:38,952 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be4cb5da32eefc543e09e80b426b6938, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:38,952 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4bf848f8297d89de7b203fcc6678eb5, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:38,952 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1202ee0cad3e428893fcdbc630d4f9b9, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:38,952 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=653572572648db72daea94dd65cfb7f8, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:38,952 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5fe5dc5eb9193cba5339ee508dff867, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:38,952 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=145c6ba0291376a96e3a37f6b274f63b, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:38,952 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5162d3c0b3255050cbeed27e4456e5c2, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:38,952 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d3ce5f64ef1889a3856db9d8c0f63a72, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:38,953 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60c948f12e02b6ab0066426b6aceea80, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=5 2024-12-12T06:12:38,953 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=764fc3b5431e53dfb445323d0bfd39a6, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:38,953 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=269d56dd3a6671f5c9a30401b73ade89, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:38,953 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae428798a6433b4d48a205ab3d14ba42, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:38,953 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=269eed2924060312949b51c10150f409, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:38,953 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-12T06:12:38,953 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733984018953 2024-12-12T06:12:38,953 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733984078953 2024-12-12T06:12:38,953 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-12-12T06:12:38,953 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-12T06:12:38,953 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-12T06:12:38,953 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-12T06:12:38,953 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-12T06:12:38,954 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-12T06:12:38,954 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-12T06:12:38,954 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-12T06:12:38,954 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-12T06:12:38,955 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-12T06:12:38,957 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-12T06:12:38,957 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-12T06:12:38,957 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-12T06:12:38,957 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:38,957 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-12T06:12:38,957 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:38,957 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:38,957 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-12T06:12:38,958 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:38,958 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:38,958 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:38,958 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-12T06:12:38,958 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:38,958 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-12T06:12:38,958 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:38,958 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-12T06:12:38,958 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-12T06:12:38,958 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-12T06:12:38,958 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:38,958 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-12T06:12:38,958 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-12T06:12:38,959 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-12T06:12:38,959 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=60c948f12e02b6ab0066426b6aceea80, REOPEN/MOVE 2024-12-12T06:12:38,961 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true 2024-12-12T06:12:38,961 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:38,961 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:38,961 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-12T06:12:38,961 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:38,961 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:38,961 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:38,961 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN 2024-12-12T06:12:38,961 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN 2024-12-12T06:12:38,961 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN 2024-12-12T06:12:38,961 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8545c1fe97252abe3447dcfaebd5cfa5, ASSIGN 2024-12-12T06:12:38,961 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1202ee0cad3e428893fcdbc630d4f9b9, ASSIGN 2024-12-12T06:12:38,961 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae428798a6433b4d48a205ab3d14ba42, ASSIGN 2024-12-12T06:12:38,961 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4584d26a35c36968de829b34ac08fb5d, ASSIGN 2024-12-12T06:12:38,961 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d86220c39cb75f305e7ed4d4c9f1191c, ASSIGN 2024-12-12T06:12:38,961 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure ae428798a6433b4d48a205ab3d14ba42, server=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:38,961 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:38,961 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure d86220c39cb75f305e7ed4d4c9f1191c, server=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:38,961 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 145c6ba0291376a96e3a37f6b274f63b, server=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:38,961 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4584d26a35c36968de829b34ac08fb5d, server=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:38,961 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:38,961 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:38,961 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:38,961 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:38,962 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:38,962 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:38,962 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:38,962 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:38,962 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN 2024-12-12T06:12:38,962 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:38,962 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN 2024-12-12T06:12:38,962 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:38,962 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN 2024-12-12T06:12:38,962 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8545c1fe97252abe3447dcfaebd5cfa5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:38,962 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8545c1fe97252abe3447dcfaebd5cfa5, ASSIGN 2024-12-12T06:12:38,962 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1202ee0cad3e428893fcdbc630d4f9b9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:38,962 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1202ee0cad3e428893fcdbc630d4f9b9, ASSIGN 2024-12-12T06:12:38,962 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae428798a6433b4d48a205ab3d14ba42, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:38,962 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae428798a6433b4d48a205ab3d14ba42, ASSIGN 2024-12-12T06:12:38,962 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4584d26a35c36968de829b34ac08fb5d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:38,962 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4584d26a35c36968de829b34ac08fb5d, ASSIGN 2024-12-12T06:12:38,962 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d86220c39cb75f305e7ed4d4c9f1191c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:38,962 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d86220c39cb75f305e7ed4d4c9f1191c, ASSIGN 2024-12-12T06:12:38,962 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure ae428798a6433b4d48a205ab3d14ba42, server=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:38,962 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:38,962 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure d86220c39cb75f305e7ed4d4c9f1191c, server=ffbfd3107920,46199,1733983951048 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:38,962 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 145c6ba0291376a96e3a37f6b274f63b, server=ffbfd3107920,46199,1733983951048 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:38,962 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4584d26a35c36968de829b34ac08fb5d, server=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:38,962 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-12-12T06:12:38,963 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=65 2024-12-12T06:12:38,963 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ffbfd3107920,42275,1733983951224, table=hbase:meta, region=1588230740 2024-12-12T06:12:38,963 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ffbfd3107920,42275,1733983951224, state=OPEN 2024-12-12T06:12:38,972 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45209-0x10018b511340000, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:38,972 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42275-0x10018b511340003, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:38,972 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46199-0x10018b511340001, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:38,973 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:38,973 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:38,973 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:38,973 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN to state=OFFLINE, location=null, table=test, region=145c6ba0291376a96e3a37f6b274f63b to restore RIT 2024-12-12T06:12:38,973 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN to state=OFFLINE, location=null, table=test, region=57995ad3e527ada56b846564833341d1 to restore RIT 2024-12-12T06:12:38,973 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN to state=OFFLINE, location=null, table=test, region=5d34bdb8a1414fb25a475fb97fc440b4 to restore RIT 2024-12-12T06:12:38,973 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8545c1fe97252abe3447dcfaebd5cfa5, ASSIGN to state=OFFLINE, location=null, table=test, region=8545c1fe97252abe3447dcfaebd5cfa5 to restore RIT 2024-12-12T06:12:38,973 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1202ee0cad3e428893fcdbc630d4f9b9, ASSIGN to state=OFFLINE, location=null, table=test, region=1202ee0cad3e428893fcdbc630d4f9b9 to restore RIT 2024-12-12T06:12:38,973 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae428798a6433b4d48a205ab3d14ba42, ASSIGN to state=OFFLINE, location=null, table=test, region=ae428798a6433b4d48a205ab3d14ba42 to restore RIT 2024-12-12T06:12:38,973 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4584d26a35c36968de829b34ac08fb5d, ASSIGN to state=OFFLINE, location=null, table=test, region=4584d26a35c36968de829b34ac08fb5d to restore RIT 2024-12-12T06:12:38,973 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d86220c39cb75f305e7ed4d4c9f1191c, ASSIGN to state=OFFLINE, location=null, table=test, region=d86220c39cb75f305e7ed4d4c9f1191c to restore RIT 2024-12-12T06:12:38,974 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733983988974 2024-12-12T06:12:38,974 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-12T06:12:38,974 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-12T06:12:38,975 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-12T06:12:38,976 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-12T06:12:38,976 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=78, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 145c6ba0291376a96e3a37f6b274f63b, server=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:38,976 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-12T06:12:38,976 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=78, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 145c6ba0291376a96e3a37f6b274f63b, server=ffbfd3107920,46199,1733983951048 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-12T06:12:38,980 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8545c1fe97252abe3447dcfaebd5cfa5, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:38,980 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16838c5df0ab4e3938ad84e9a080def9, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:38,980 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2eeaafb9a1964f4c6899fa5dde05a463, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:38,981 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c98bb859cc9e047b211ddb4c5b967ce5, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:38,981 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b935ee7eeea02b161c0ebc0d05527e91, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:38,981 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4584d26a35c36968de829b34ac08fb5d, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:38,981 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b717f65da42dacb67935407f094e8467, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:38,981 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f2b6a6b11f14670aca9dccceb20d9630, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:38,981 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57995ad3e527ada56b846564833341d1, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:38,982 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c77db308a7c1c57fe9ef945e17e28127, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:38,982 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=782c52424de7d71cb43b8fbfe8e89f7f, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:38,982 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d34bdb8a1414fb25a475fb97fc440b4, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:38,982 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d86220c39cb75f305e7ed4d4c9f1191c, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:38,982 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be4cb5da32eefc543e09e80b426b6938, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:38,982 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4bf848f8297d89de7b203fcc6678eb5, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:38,982 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1202ee0cad3e428893fcdbc630d4f9b9, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:38,983 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=653572572648db72daea94dd65cfb7f8, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:38,983 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5fe5dc5eb9193cba5339ee508dff867, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:38,983 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=145c6ba0291376a96e3a37f6b274f63b, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:38,983 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5162d3c0b3255050cbeed27e4456e5c2, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:38,983 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d3ce5f64ef1889a3856db9d8c0f63a72, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:38,984 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60c948f12e02b6ab0066426b6aceea80, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=5 2024-12-12T06:12:38,984 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=764fc3b5431e53dfb445323d0bfd39a6, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:38,984 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=269d56dd3a6671f5c9a30401b73ade89, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:38,984 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae428798a6433b4d48a205ab3d14ba42, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:38,984 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=269eed2924060312949b51c10150f409, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:38,985 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-12T06:12:38,985 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733984018985 2024-12-12T06:12:38,985 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733984078985 2024-12-12T06:12:38,985 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-12-12T06:12:38,985 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-12T06:12:38,985 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-12T06:12:38,985 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-12T06:12:38,985 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-12T06:12:38,985 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-12T06:12:38,986 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-12T06:12:38,986 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-12T06:12:38,986 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-12T06:12:38,987 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-12T06:12:38,990 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-12T06:12:38,991 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-12T06:12:38,991 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-12T06:12:38,991 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:38,991 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-12T06:12:38,991 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:38,991 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:38,991 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-12T06:12:38,991 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:38,991 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:38,991 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:38,991 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-12T06:12:38,991 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:38,991 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:38,991 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-12T06:12:38,992 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-12T06:12:38,992 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-12T06:12:38,992 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:38,992 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-12T06:12:38,992 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-12T06:12:38,993 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-12T06:12:38,993 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=60c948f12e02b6ab0066426b6aceea80, REOPEN/MOVE 2024-12-12T06:12:38,995 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true 2024-12-12T06:12:38,996 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:38,996 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:38,996 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-12T06:12:38,996 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:38,996 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:38,996 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:38,996 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN 2024-12-12T06:12:38,996 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN 2024-12-12T06:12:38,996 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN 2024-12-12T06:12:38,996 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8545c1fe97252abe3447dcfaebd5cfa5, ASSIGN 2024-12-12T06:12:38,996 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1202ee0cad3e428893fcdbc630d4f9b9, ASSIGN 2024-12-12T06:12:38,996 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae428798a6433b4d48a205ab3d14ba42, ASSIGN 2024-12-12T06:12:38,996 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4584d26a35c36968de829b34ac08fb5d, ASSIGN 2024-12-12T06:12:38,996 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d86220c39cb75f305e7ed4d4c9f1191c, ASSIGN 2024-12-12T06:12:38,996 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure ae428798a6433b4d48a205ab3d14ba42, server=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:38,996 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:38,996 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure d86220c39cb75f305e7ed4d4c9f1191c, server=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:38,996 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 145c6ba0291376a96e3a37f6b274f63b, server=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:38,996 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:38,996 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:38,996 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:38,997 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:38,997 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:38,997 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:38,997 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:38,997 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:38,997 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN 2024-12-12T06:12:38,997 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:38,997 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN 2024-12-12T06:12:38,997 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:38,997 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN 2024-12-12T06:12:38,997 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8545c1fe97252abe3447dcfaebd5cfa5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:38,997 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8545c1fe97252abe3447dcfaebd5cfa5, ASSIGN 2024-12-12T06:12:38,997 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1202ee0cad3e428893fcdbc630d4f9b9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:38,997 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1202ee0cad3e428893fcdbc630d4f9b9, ASSIGN 2024-12-12T06:12:38,997 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae428798a6433b4d48a205ab3d14ba42, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:38,997 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae428798a6433b4d48a205ab3d14ba42, ASSIGN 2024-12-12T06:12:38,997 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4584d26a35c36968de829b34ac08fb5d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:38,997 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4584d26a35c36968de829b34ac08fb5d, ASSIGN 2024-12-12T06:12:38,997 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d86220c39cb75f305e7ed4d4c9f1191c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:38,997 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d86220c39cb75f305e7ed4d4c9f1191c, ASSIGN 2024-12-12T06:12:38,997 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure ae428798a6433b4d48a205ab3d14ba42, server=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:38,997 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:38,997 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure d86220c39cb75f305e7ed4d4c9f1191c, server=ffbfd3107920,46199,1733983951048 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:38,997 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 145c6ba0291376a96e3a37f6b274f63b, server=ffbfd3107920,46199,1733983951048 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:38,998 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 10 msec 2024-12-12T06:12:38,999 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=65 2024-12-12T06:12:38,999 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ffbfd3107920,42275,1733983951224, table=hbase:meta, region=1588230740 2024-12-12T06:12:38,999 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ffbfd3107920,42275,1733983951224, state=OPEN 2024-12-12T06:12:39,009 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46199-0x10018b511340001, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:39,009 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42275-0x10018b511340003, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:39,009 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45209-0x10018b511340000, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:39,009 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:39,009 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:39,009 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:39,010 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN to state=OFFLINE, location=null, table=test, region=145c6ba0291376a96e3a37f6b274f63b to restore RIT 2024-12-12T06:12:39,010 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN to state=OFFLINE, location=null, table=test, region=57995ad3e527ada56b846564833341d1 to restore RIT 2024-12-12T06:12:39,010 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN to state=OFFLINE, location=null, table=test, region=5d34bdb8a1414fb25a475fb97fc440b4 to restore RIT 2024-12-12T06:12:39,010 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8545c1fe97252abe3447dcfaebd5cfa5, ASSIGN to state=OFFLINE, location=null, table=test, region=8545c1fe97252abe3447dcfaebd5cfa5 to restore RIT 2024-12-12T06:12:39,010 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1202ee0cad3e428893fcdbc630d4f9b9, ASSIGN to state=OFFLINE, location=null, table=test, region=1202ee0cad3e428893fcdbc630d4f9b9 to restore RIT 2024-12-12T06:12:39,010 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae428798a6433b4d48a205ab3d14ba42, ASSIGN to state=OFFLINE, location=null, table=test, region=ae428798a6433b4d48a205ab3d14ba42 to restore RIT 2024-12-12T06:12:39,010 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4584d26a35c36968de829b34ac08fb5d, ASSIGN to state=OFFLINE, location=null, table=test, region=4584d26a35c36968de829b34ac08fb5d to restore RIT 2024-12-12T06:12:39,010 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d86220c39cb75f305e7ed4d4c9f1191c, ASSIGN to state=OFFLINE, location=null, table=test, region=d86220c39cb75f305e7ed4d4c9f1191c to restore RIT 2024-12-12T06:12:39,011 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733983989011 2024-12-12T06:12:39,011 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-12T06:12:39,011 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-12T06:12:39,012 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-12T06:12:39,013 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-12T06:12:39,013 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure d86220c39cb75f305e7ed4d4c9f1191c, server=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:39,013 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-12T06:12:39,013 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure d86220c39cb75f305e7ed4d4c9f1191c, server=ffbfd3107920,46199,1733983951048 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-12T06:12:39,017 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8545c1fe97252abe3447dcfaebd5cfa5, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,017 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16838c5df0ab4e3938ad84e9a080def9, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,017 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2eeaafb9a1964f4c6899fa5dde05a463, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,018 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c98bb859cc9e047b211ddb4c5b967ce5, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,018 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b935ee7eeea02b161c0ebc0d05527e91, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,018 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4584d26a35c36968de829b34ac08fb5d, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,018 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b717f65da42dacb67935407f094e8467, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,018 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f2b6a6b11f14670aca9dccceb20d9630, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,018 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57995ad3e527ada56b846564833341d1, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,018 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c77db308a7c1c57fe9ef945e17e28127, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,018 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=782c52424de7d71cb43b8fbfe8e89f7f, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,018 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d34bdb8a1414fb25a475fb97fc440b4, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,018 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d86220c39cb75f305e7ed4d4c9f1191c, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,019 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be4cb5da32eefc543e09e80b426b6938, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,019 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4bf848f8297d89de7b203fcc6678eb5, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,019 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1202ee0cad3e428893fcdbc630d4f9b9, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,019 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=653572572648db72daea94dd65cfb7f8, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,019 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5fe5dc5eb9193cba5339ee508dff867, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,019 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=145c6ba0291376a96e3a37f6b274f63b, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,019 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5162d3c0b3255050cbeed27e4456e5c2, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,019 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d3ce5f64ef1889a3856db9d8c0f63a72, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,020 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60c948f12e02b6ab0066426b6aceea80, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=5 2024-12-12T06:12:39,020 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=764fc3b5431e53dfb445323d0bfd39a6, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,020 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=269d56dd3a6671f5c9a30401b73ade89, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,020 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae428798a6433b4d48a205ab3d14ba42, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,020 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=269eed2924060312949b51c10150f409, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,020 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-12T06:12:39,020 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733984019020 2024-12-12T06:12:39,020 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733984079020 2024-12-12T06:12:39,020 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-12-12T06:12:39,020 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-12T06:12:39,020 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-12T06:12:39,020 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-12T06:12:39,020 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-12T06:12:39,021 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-12T06:12:39,021 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-12T06:12:39,021 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-12T06:12:39,021 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-12T06:12:39,022 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-12T06:12:39,023 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-12T06:12:39,023 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-12T06:12:39,024 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-12T06:12:39,024 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,024 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,024 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,024 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-12T06:12:39,024 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,024 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,024 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,024 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-12T06:12:39,024 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,024 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,024 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-12T06:12:39,024 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-12T06:12:39,024 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-12T06:12:39,024 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,024 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-12T06:12:39,024 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-12T06:12:39,025 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-12T06:12:39,025 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=60c948f12e02b6ab0066426b6aceea80, REOPEN/MOVE 2024-12-12T06:12:39,026 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true 2024-12-12T06:12:39,026 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:39,026 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:39,026 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-12T06:12:39,026 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:39,026 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:39,026 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:39,026 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN 2024-12-12T06:12:39,026 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN 2024-12-12T06:12:39,026 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN 2024-12-12T06:12:39,026 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8545c1fe97252abe3447dcfaebd5cfa5, ASSIGN 2024-12-12T06:12:39,026 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1202ee0cad3e428893fcdbc630d4f9b9, ASSIGN 2024-12-12T06:12:39,026 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae428798a6433b4d48a205ab3d14ba42, ASSIGN 2024-12-12T06:12:39,026 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4584d26a35c36968de829b34ac08fb5d, ASSIGN 2024-12-12T06:12:39,026 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d86220c39cb75f305e7ed4d4c9f1191c, ASSIGN 2024-12-12T06:12:39,026 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure ae428798a6433b4d48a205ab3d14ba42, server=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:39,026 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:39,026 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure d86220c39cb75f305e7ed4d4c9f1191c, server=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:39,026 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,027 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,027 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,027 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,027 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,027 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,027 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,027 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,027 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN 2024-12-12T06:12:39,027 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,027 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN 2024-12-12T06:12:39,027 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,027 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN 2024-12-12T06:12:39,027 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8545c1fe97252abe3447dcfaebd5cfa5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,027 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8545c1fe97252abe3447dcfaebd5cfa5, ASSIGN 2024-12-12T06:12:39,027 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1202ee0cad3e428893fcdbc630d4f9b9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,027 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1202ee0cad3e428893fcdbc630d4f9b9, ASSIGN 2024-12-12T06:12:39,027 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae428798a6433b4d48a205ab3d14ba42, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,027 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae428798a6433b4d48a205ab3d14ba42, ASSIGN 2024-12-12T06:12:39,027 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4584d26a35c36968de829b34ac08fb5d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,027 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4584d26a35c36968de829b34ac08fb5d, ASSIGN 2024-12-12T06:12:39,027 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d86220c39cb75f305e7ed4d4c9f1191c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,027 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d86220c39cb75f305e7ed4d4c9f1191c, ASSIGN 2024-12-12T06:12:39,027 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure ae428798a6433b4d48a205ab3d14ba42, server=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,027 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,027 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure d86220c39cb75f305e7ed4d4c9f1191c, server=ffbfd3107920,46199,1733983951048 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,027 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-12-12T06:12:39,028 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=65 2024-12-12T06:12:39,028 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ffbfd3107920,42275,1733983951224, table=hbase:meta, region=1588230740 2024-12-12T06:12:39,028 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ffbfd3107920,42275,1733983951224, state=OPEN 2024-12-12T06:12:39,039 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42275-0x10018b511340003, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:39,039 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46199-0x10018b511340001, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:39,039 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45209-0x10018b511340000, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:39,039 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:39,039 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:39,039 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:39,040 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN to state=OFFLINE, location=null, table=test, region=145c6ba0291376a96e3a37f6b274f63b to restore RIT 2024-12-12T06:12:39,040 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN to state=OFFLINE, location=null, table=test, region=57995ad3e527ada56b846564833341d1 to restore RIT 2024-12-12T06:12:39,040 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN to state=OFFLINE, location=null, table=test, region=5d34bdb8a1414fb25a475fb97fc440b4 to restore RIT 2024-12-12T06:12:39,040 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8545c1fe97252abe3447dcfaebd5cfa5, ASSIGN to state=OFFLINE, location=null, table=test, region=8545c1fe97252abe3447dcfaebd5cfa5 to restore RIT 2024-12-12T06:12:39,040 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1202ee0cad3e428893fcdbc630d4f9b9, ASSIGN to state=OFFLINE, location=null, table=test, region=1202ee0cad3e428893fcdbc630d4f9b9 to restore RIT 2024-12-12T06:12:39,040 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae428798a6433b4d48a205ab3d14ba42, ASSIGN to state=OFFLINE, location=null, table=test, region=ae428798a6433b4d48a205ab3d14ba42 to restore RIT 2024-12-12T06:12:39,040 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4584d26a35c36968de829b34ac08fb5d, ASSIGN to state=OFFLINE, location=null, table=test, region=4584d26a35c36968de829b34ac08fb5d to restore RIT 2024-12-12T06:12:39,040 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d86220c39cb75f305e7ed4d4c9f1191c, ASSIGN to state=OFFLINE, location=null, table=test, region=d86220c39cb75f305e7ed4d4c9f1191c to restore RIT 2024-12-12T06:12:39,041 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733983989040 2024-12-12T06:12:39,041 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-12T06:12:39,041 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-12T06:12:39,042 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-12T06:12:39,043 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-12T06:12:39,043 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=76, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:39,043 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-12T06:12:39,043 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=76, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-12T06:12:39,046 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8545c1fe97252abe3447dcfaebd5cfa5, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,047 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16838c5df0ab4e3938ad84e9a080def9, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,047 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2eeaafb9a1964f4c6899fa5dde05a463, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,047 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c98bb859cc9e047b211ddb4c5b967ce5, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,047 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b935ee7eeea02b161c0ebc0d05527e91, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,047 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4584d26a35c36968de829b34ac08fb5d, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,047 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b717f65da42dacb67935407f094e8467, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,047 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f2b6a6b11f14670aca9dccceb20d9630, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,047 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57995ad3e527ada56b846564833341d1, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,047 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c77db308a7c1c57fe9ef945e17e28127, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,048 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=782c52424de7d71cb43b8fbfe8e89f7f, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,048 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d34bdb8a1414fb25a475fb97fc440b4, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,048 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d86220c39cb75f305e7ed4d4c9f1191c, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,048 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be4cb5da32eefc543e09e80b426b6938, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,048 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4bf848f8297d89de7b203fcc6678eb5, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,048 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1202ee0cad3e428893fcdbc630d4f9b9, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,048 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=653572572648db72daea94dd65cfb7f8, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,048 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5fe5dc5eb9193cba5339ee508dff867, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,048 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=145c6ba0291376a96e3a37f6b274f63b, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,048 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5162d3c0b3255050cbeed27e4456e5c2, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,049 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d3ce5f64ef1889a3856db9d8c0f63a72, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,049 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60c948f12e02b6ab0066426b6aceea80, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=5 2024-12-12T06:12:39,049 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=764fc3b5431e53dfb445323d0bfd39a6, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,049 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=269d56dd3a6671f5c9a30401b73ade89, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,049 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae428798a6433b4d48a205ab3d14ba42, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,049 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=269eed2924060312949b51c10150f409, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,049 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-12T06:12:39,049 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733984019049 2024-12-12T06:12:39,049 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733984079049 2024-12-12T06:12:39,049 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-12-12T06:12:39,049 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-12T06:12:39,049 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-12T06:12:39,049 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-12T06:12:39,050 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-12T06:12:39,050 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-12T06:12:39,050 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-12T06:12:39,050 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-12T06:12:39,050 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-12T06:12:39,051 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-12T06:12:39,052 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-12T06:12:39,052 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-12T06:12:39,052 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-12T06:12:39,052 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,052 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,053 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,053 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-12T06:12:39,053 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,053 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,053 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,053 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-12T06:12:39,053 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,053 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,053 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-12T06:12:39,053 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-12T06:12:39,053 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,053 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-12T06:12:39,053 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-12T06:12:39,053 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-12T06:12:39,054 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=60c948f12e02b6ab0066426b6aceea80, REOPEN/MOVE 2024-12-12T06:12:39,055 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true 2024-12-12T06:12:39,055 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:39,055 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:39,055 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-12T06:12:39,055 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:39,055 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:39,055 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:39,055 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN 2024-12-12T06:12:39,055 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN 2024-12-12T06:12:39,055 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN 2024-12-12T06:12:39,055 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8545c1fe97252abe3447dcfaebd5cfa5, ASSIGN 2024-12-12T06:12:39,055 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1202ee0cad3e428893fcdbc630d4f9b9, ASSIGN 2024-12-12T06:12:39,055 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae428798a6433b4d48a205ab3d14ba42, ASSIGN 2024-12-12T06:12:39,055 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4584d26a35c36968de829b34ac08fb5d, ASSIGN 2024-12-12T06:12:39,055 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d86220c39cb75f305e7ed4d4c9f1191c, ASSIGN 2024-12-12T06:12:39,055 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure ae428798a6433b4d48a205ab3d14ba42, server=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:39,055 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:39,055 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,055 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,055 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,055 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,055 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,055 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,055 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,055 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,055 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN 2024-12-12T06:12:39,056 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,056 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN 2024-12-12T06:12:39,056 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,056 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN 2024-12-12T06:12:39,056 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8545c1fe97252abe3447dcfaebd5cfa5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,056 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8545c1fe97252abe3447dcfaebd5cfa5, ASSIGN 2024-12-12T06:12:39,056 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1202ee0cad3e428893fcdbc630d4f9b9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,056 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1202ee0cad3e428893fcdbc630d4f9b9, ASSIGN 2024-12-12T06:12:39,056 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae428798a6433b4d48a205ab3d14ba42, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,056 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae428798a6433b4d48a205ab3d14ba42, ASSIGN 2024-12-12T06:12:39,056 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4584d26a35c36968de829b34ac08fb5d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,056 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4584d26a35c36968de829b34ac08fb5d, ASSIGN 2024-12-12T06:12:39,056 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d86220c39cb75f305e7ed4d4c9f1191c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,056 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d86220c39cb75f305e7ed4d4c9f1191c, ASSIGN 2024-12-12T06:12:39,056 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure ae428798a6433b4d48a205ab3d14ba42, server=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,056 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,056 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-12-12T06:12:39,057 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=65 2024-12-12T06:12:39,057 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ffbfd3107920,42275,1733983951224, table=hbase:meta, region=1588230740 2024-12-12T06:12:39,057 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ffbfd3107920,42275,1733983951224, state=OPEN 2024-12-12T06:12:39,068 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42275-0x10018b511340003, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:39,068 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45209-0x10018b511340000, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:39,068 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46199-0x10018b511340001, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:39,068 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:39,068 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:39,068 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:39,068 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN to state=OFFLINE, location=null, table=test, region=145c6ba0291376a96e3a37f6b274f63b to restore RIT 2024-12-12T06:12:39,068 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN to state=OFFLINE, location=null, table=test, region=57995ad3e527ada56b846564833341d1 to restore RIT 2024-12-12T06:12:39,069 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN to state=OFFLINE, location=null, table=test, region=5d34bdb8a1414fb25a475fb97fc440b4 to restore RIT 2024-12-12T06:12:39,069 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8545c1fe97252abe3447dcfaebd5cfa5, ASSIGN to state=OFFLINE, location=null, table=test, region=8545c1fe97252abe3447dcfaebd5cfa5 to restore RIT 2024-12-12T06:12:39,069 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1202ee0cad3e428893fcdbc630d4f9b9, ASSIGN to state=OFFLINE, location=null, table=test, region=1202ee0cad3e428893fcdbc630d4f9b9 to restore RIT 2024-12-12T06:12:39,069 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae428798a6433b4d48a205ab3d14ba42, ASSIGN to state=OFFLINE, location=null, table=test, region=ae428798a6433b4d48a205ab3d14ba42 to restore RIT 2024-12-12T06:12:39,069 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4584d26a35c36968de829b34ac08fb5d, ASSIGN to state=OFFLINE, location=null, table=test, region=4584d26a35c36968de829b34ac08fb5d to restore RIT 2024-12-12T06:12:39,069 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d86220c39cb75f305e7ed4d4c9f1191c, ASSIGN to state=OFFLINE, location=null, table=test, region=d86220c39cb75f305e7ed4d4c9f1191c to restore RIT 2024-12-12T06:12:39,069 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733983989069 2024-12-12T06:12:39,069 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-12T06:12:39,069 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-12T06:12:39,071 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-12T06:12:39,072 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-12T06:12:39,072 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=75, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure ae428798a6433b4d48a205ab3d14ba42, server=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:39,072 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-12T06:12:39,072 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=75, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure ae428798a6433b4d48a205ab3d14ba42, server=ffbfd3107920,42275,1733983951224 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-12T06:12:39,076 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8545c1fe97252abe3447dcfaebd5cfa5, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,076 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16838c5df0ab4e3938ad84e9a080def9, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,076 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2eeaafb9a1964f4c6899fa5dde05a463, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,076 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c98bb859cc9e047b211ddb4c5b967ce5, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,077 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b935ee7eeea02b161c0ebc0d05527e91, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,077 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4584d26a35c36968de829b34ac08fb5d, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,077 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b717f65da42dacb67935407f094e8467, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,077 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f2b6a6b11f14670aca9dccceb20d9630, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,077 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57995ad3e527ada56b846564833341d1, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,077 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c77db308a7c1c57fe9ef945e17e28127, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,077 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=782c52424de7d71cb43b8fbfe8e89f7f, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,078 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d34bdb8a1414fb25a475fb97fc440b4, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,078 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d86220c39cb75f305e7ed4d4c9f1191c, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,078 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be4cb5da32eefc543e09e80b426b6938, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,078 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4bf848f8297d89de7b203fcc6678eb5, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,078 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1202ee0cad3e428893fcdbc630d4f9b9, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,078 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=653572572648db72daea94dd65cfb7f8, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,078 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5fe5dc5eb9193cba5339ee508dff867, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,079 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=145c6ba0291376a96e3a37f6b274f63b, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,079 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5162d3c0b3255050cbeed27e4456e5c2, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,079 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d3ce5f64ef1889a3856db9d8c0f63a72, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,079 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60c948f12e02b6ab0066426b6aceea80, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=5 2024-12-12T06:12:39,079 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=764fc3b5431e53dfb445323d0bfd39a6, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,079 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=269d56dd3a6671f5c9a30401b73ade89, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,080 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae428798a6433b4d48a205ab3d14ba42, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,080 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=269eed2924060312949b51c10150f409, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,080 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-12T06:12:39,080 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733984019080 2024-12-12T06:12:39,080 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733984079080 2024-12-12T06:12:39,080 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-12-12T06:12:39,080 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-12T06:12:39,080 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-12T06:12:39,080 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-12T06:12:39,080 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-12T06:12:39,081 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-12T06:12:39,081 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-12T06:12:39,081 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-12T06:12:39,081 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-12T06:12:39,082 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-12T06:12:39,084 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-12T06:12:39,084 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-12T06:12:39,084 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-12T06:12:39,084 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,084 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,084 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,084 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,084 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,084 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,084 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-12T06:12:39,084 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,084 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,084 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-12T06:12:39,084 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-12T06:12:39,084 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,084 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-12T06:12:39,085 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-12T06:12:39,085 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-12T06:12:39,086 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=60c948f12e02b6ab0066426b6aceea80, REOPEN/MOVE 2024-12-12T06:12:39,088 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true 2024-12-12T06:12:39,088 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:39,088 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:39,088 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-12T06:12:39,088 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:39,088 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:39,088 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:39,088 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN 2024-12-12T06:12:39,088 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN 2024-12-12T06:12:39,088 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN 2024-12-12T06:12:39,088 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8545c1fe97252abe3447dcfaebd5cfa5, ASSIGN 2024-12-12T06:12:39,088 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1202ee0cad3e428893fcdbc630d4f9b9, ASSIGN 2024-12-12T06:12:39,088 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae428798a6433b4d48a205ab3d14ba42, ASSIGN 2024-12-12T06:12:39,088 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4584d26a35c36968de829b34ac08fb5d, ASSIGN 2024-12-12T06:12:39,088 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d86220c39cb75f305e7ed4d4c9f1191c, ASSIGN 2024-12-12T06:12:39,088 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure ae428798a6433b4d48a205ab3d14ba42, server=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:39,088 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,089 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,089 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,089 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,089 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,089 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,089 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,089 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,089 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN 2024-12-12T06:12:39,089 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,089 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN 2024-12-12T06:12:39,089 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,089 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN 2024-12-12T06:12:39,089 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8545c1fe97252abe3447dcfaebd5cfa5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,089 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8545c1fe97252abe3447dcfaebd5cfa5, ASSIGN 2024-12-12T06:12:39,089 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1202ee0cad3e428893fcdbc630d4f9b9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,089 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1202ee0cad3e428893fcdbc630d4f9b9, ASSIGN 2024-12-12T06:12:39,089 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae428798a6433b4d48a205ab3d14ba42, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,089 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae428798a6433b4d48a205ab3d14ba42, ASSIGN 2024-12-12T06:12:39,089 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4584d26a35c36968de829b34ac08fb5d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,089 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4584d26a35c36968de829b34ac08fb5d, ASSIGN 2024-12-12T06:12:39,089 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d86220c39cb75f305e7ed4d4c9f1191c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,089 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d86220c39cb75f305e7ed4d4c9f1191c, ASSIGN 2024-12-12T06:12:39,089 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure ae428798a6433b4d48a205ab3d14ba42, server=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,090 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-12-12T06:12:39,090 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=65 2024-12-12T06:12:39,090 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ffbfd3107920,42275,1733983951224, table=hbase:meta, region=1588230740 2024-12-12T06:12:39,091 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ffbfd3107920,42275,1733983951224, state=OPEN 2024-12-12T06:12:39,101 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46199-0x10018b511340001, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:39,101 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42275-0x10018b511340003, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:39,101 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45209-0x10018b511340000, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:39,101 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:39,101 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:39,101 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:39,102 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN to state=OFFLINE, location=null, table=test, region=145c6ba0291376a96e3a37f6b274f63b to restore RIT 2024-12-12T06:12:39,102 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN to state=OFFLINE, location=null, table=test, region=57995ad3e527ada56b846564833341d1 to restore RIT 2024-12-12T06:12:39,102 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN to state=OFFLINE, location=null, table=test, region=5d34bdb8a1414fb25a475fb97fc440b4 to restore RIT 2024-12-12T06:12:39,102 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8545c1fe97252abe3447dcfaebd5cfa5, ASSIGN to state=OFFLINE, location=null, table=test, region=8545c1fe97252abe3447dcfaebd5cfa5 to restore RIT 2024-12-12T06:12:39,102 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1202ee0cad3e428893fcdbc630d4f9b9, ASSIGN to state=OFFLINE, location=null, table=test, region=1202ee0cad3e428893fcdbc630d4f9b9 to restore RIT 2024-12-12T06:12:39,102 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae428798a6433b4d48a205ab3d14ba42, ASSIGN to state=OFFLINE, location=null, table=test, region=ae428798a6433b4d48a205ab3d14ba42 to restore RIT 2024-12-12T06:12:39,102 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4584d26a35c36968de829b34ac08fb5d, ASSIGN to state=OFFLINE, location=null, table=test, region=4584d26a35c36968de829b34ac08fb5d to restore RIT 2024-12-12T06:12:39,102 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d86220c39cb75f305e7ed4d4c9f1191c, ASSIGN to state=OFFLINE, location=null, table=test, region=d86220c39cb75f305e7ed4d4c9f1191c to restore RIT 2024-12-12T06:12:39,102 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733983989102 2024-12-12T06:12:39,102 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-12T06:12:39,103 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-12T06:12:39,104 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-12T06:12:39,105 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-12T06:12:39,105 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d86220c39cb75f305e7ed4d4c9f1191c, ASSIGN 2024-12-12T06:12:39,105 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-12T06:12:39,105 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d86220c39cb75f305e7ed4d4c9f1191c, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-12T06:12:39,107 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8545c1fe97252abe3447dcfaebd5cfa5, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,107 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16838c5df0ab4e3938ad84e9a080def9, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,107 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2eeaafb9a1964f4c6899fa5dde05a463, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,107 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c98bb859cc9e047b211ddb4c5b967ce5, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,107 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b935ee7eeea02b161c0ebc0d05527e91, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,108 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4584d26a35c36968de829b34ac08fb5d, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,108 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b717f65da42dacb67935407f094e8467, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,108 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f2b6a6b11f14670aca9dccceb20d9630, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,108 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57995ad3e527ada56b846564833341d1, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,108 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c77db308a7c1c57fe9ef945e17e28127, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,108 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=782c52424de7d71cb43b8fbfe8e89f7f, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,108 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d34bdb8a1414fb25a475fb97fc440b4, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,108 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d86220c39cb75f305e7ed4d4c9f1191c, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,108 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be4cb5da32eefc543e09e80b426b6938, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,108 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4bf848f8297d89de7b203fcc6678eb5, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,109 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1202ee0cad3e428893fcdbc630d4f9b9, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,109 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=653572572648db72daea94dd65cfb7f8, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,109 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5fe5dc5eb9193cba5339ee508dff867, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,109 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=145c6ba0291376a96e3a37f6b274f63b, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,109 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5162d3c0b3255050cbeed27e4456e5c2, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,109 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d3ce5f64ef1889a3856db9d8c0f63a72, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,109 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60c948f12e02b6ab0066426b6aceea80, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=5 2024-12-12T06:12:39,109 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=764fc3b5431e53dfb445323d0bfd39a6, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,109 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=269d56dd3a6671f5c9a30401b73ade89, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,109 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae428798a6433b4d48a205ab3d14ba42, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,109 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=269eed2924060312949b51c10150f409, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,110 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-12T06:12:39,110 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733984019110 2024-12-12T06:12:39,110 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733984079110 2024-12-12T06:12:39,110 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-12-12T06:12:39,110 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-12T06:12:39,110 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-12T06:12:39,110 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-12T06:12:39,110 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-12T06:12:39,110 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-12T06:12:39,110 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-12T06:12:39,110 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-12T06:12:39,110 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-12T06:12:39,111 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-12T06:12:39,112 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-12T06:12:39,112 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-12T06:12:39,112 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-12T06:12:39,112 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,112 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,113 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,113 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,113 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,113 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,113 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,113 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,113 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-12T06:12:39,113 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-12T06:12:39,113 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,113 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-12T06:12:39,113 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-12T06:12:39,114 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-12T06:12:39,114 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=60c948f12e02b6ab0066426b6aceea80, REOPEN/MOVE 2024-12-12T06:12:39,115 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true 2024-12-12T06:12:39,115 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:39,115 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:39,115 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-12T06:12:39,115 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:39,115 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:39,115 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:39,115 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN 2024-12-12T06:12:39,115 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN 2024-12-12T06:12:39,115 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN 2024-12-12T06:12:39,115 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8545c1fe97252abe3447dcfaebd5cfa5, ASSIGN 2024-12-12T06:12:39,115 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1202ee0cad3e428893fcdbc630d4f9b9, ASSIGN 2024-12-12T06:12:39,115 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae428798a6433b4d48a205ab3d14ba42, ASSIGN 2024-12-12T06:12:39,115 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4584d26a35c36968de829b34ac08fb5d, ASSIGN 2024-12-12T06:12:39,115 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d86220c39cb75f305e7ed4d4c9f1191c, ASSIGN 2024-12-12T06:12:39,115 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,115 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,116 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,116 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,116 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,116 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,116 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,116 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,116 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN 2024-12-12T06:12:39,116 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,116 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN 2024-12-12T06:12:39,116 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,116 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN 2024-12-12T06:12:39,116 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8545c1fe97252abe3447dcfaebd5cfa5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,116 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8545c1fe97252abe3447dcfaebd5cfa5, ASSIGN 2024-12-12T06:12:39,116 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1202ee0cad3e428893fcdbc630d4f9b9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,116 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1202ee0cad3e428893fcdbc630d4f9b9, ASSIGN 2024-12-12T06:12:39,116 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae428798a6433b4d48a205ab3d14ba42, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,116 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae428798a6433b4d48a205ab3d14ba42, ASSIGN 2024-12-12T06:12:39,116 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4584d26a35c36968de829b34ac08fb5d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,116 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4584d26a35c36968de829b34ac08fb5d, ASSIGN 2024-12-12T06:12:39,116 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d86220c39cb75f305e7ed4d4c9f1191c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,116 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d86220c39cb75f305e7ed4d4c9f1191c, ASSIGN 2024-12-12T06:12:39,116 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-12T06:12:39,117 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=65 2024-12-12T06:12:39,117 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ffbfd3107920,42275,1733983951224, table=hbase:meta, region=1588230740 2024-12-12T06:12:39,117 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ffbfd3107920,42275,1733983951224, state=OPEN 2024-12-12T06:12:39,126 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46199-0x10018b511340001, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:39,126 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42275-0x10018b511340003, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:39,126 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45209-0x10018b511340000, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:39,126 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:39,126 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:39,126 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:39,127 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN to state=OFFLINE, location=null, table=test, region=145c6ba0291376a96e3a37f6b274f63b to restore RIT 2024-12-12T06:12:39,127 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN to state=OFFLINE, location=null, table=test, region=57995ad3e527ada56b846564833341d1 to restore RIT 2024-12-12T06:12:39,127 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN to state=OFFLINE, location=null, table=test, region=5d34bdb8a1414fb25a475fb97fc440b4 to restore RIT 2024-12-12T06:12:39,127 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8545c1fe97252abe3447dcfaebd5cfa5, ASSIGN to state=OFFLINE, location=null, table=test, region=8545c1fe97252abe3447dcfaebd5cfa5 to restore RIT 2024-12-12T06:12:39,127 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1202ee0cad3e428893fcdbc630d4f9b9, ASSIGN to state=OFFLINE, location=null, table=test, region=1202ee0cad3e428893fcdbc630d4f9b9 to restore RIT 2024-12-12T06:12:39,127 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae428798a6433b4d48a205ab3d14ba42, ASSIGN to state=OFFLINE, location=null, table=test, region=ae428798a6433b4d48a205ab3d14ba42 to restore RIT 2024-12-12T06:12:39,127 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4584d26a35c36968de829b34ac08fb5d, ASSIGN to state=OFFLINE, location=null, table=test, region=4584d26a35c36968de829b34ac08fb5d to restore RIT 2024-12-12T06:12:39,127 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d86220c39cb75f305e7ed4d4c9f1191c, ASSIGN to state=OFFLINE, location=null, table=test, region=d86220c39cb75f305e7ed4d4c9f1191c to restore RIT 2024-12-12T06:12:39,127 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733983989127 2024-12-12T06:12:39,127 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-12T06:12:39,127 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-12T06:12:39,128 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-12T06:12:39,129 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-12T06:12:39,129 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4584d26a35c36968de829b34ac08fb5d, ASSIGN 2024-12-12T06:12:39,129 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-12T06:12:39,129 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4584d26a35c36968de829b34ac08fb5d, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-12T06:12:39,132 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8545c1fe97252abe3447dcfaebd5cfa5, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,132 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16838c5df0ab4e3938ad84e9a080def9, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,132 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2eeaafb9a1964f4c6899fa5dde05a463, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,132 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c98bb859cc9e047b211ddb4c5b967ce5, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,132 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b935ee7eeea02b161c0ebc0d05527e91, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,133 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4584d26a35c36968de829b34ac08fb5d, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,133 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b717f65da42dacb67935407f094e8467, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,133 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f2b6a6b11f14670aca9dccceb20d9630, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,133 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57995ad3e527ada56b846564833341d1, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,133 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c77db308a7c1c57fe9ef945e17e28127, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,133 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=782c52424de7d71cb43b8fbfe8e89f7f, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,133 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d34bdb8a1414fb25a475fb97fc440b4, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,133 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d86220c39cb75f305e7ed4d4c9f1191c, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,134 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be4cb5da32eefc543e09e80b426b6938, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,134 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4bf848f8297d89de7b203fcc6678eb5, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,134 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1202ee0cad3e428893fcdbc630d4f9b9, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,134 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=653572572648db72daea94dd65cfb7f8, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,134 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5fe5dc5eb9193cba5339ee508dff867, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,134 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=145c6ba0291376a96e3a37f6b274f63b, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,134 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5162d3c0b3255050cbeed27e4456e5c2, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,134 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d3ce5f64ef1889a3856db9d8c0f63a72, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,134 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60c948f12e02b6ab0066426b6aceea80, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=5 2024-12-12T06:12:39,134 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=764fc3b5431e53dfb445323d0bfd39a6, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,135 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=269d56dd3a6671f5c9a30401b73ade89, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,135 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae428798a6433b4d48a205ab3d14ba42, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,135 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=269eed2924060312949b51c10150f409, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,135 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-12T06:12:39,135 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733984019135 2024-12-12T06:12:39,135 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733984079135 2024-12-12T06:12:39,135 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-12-12T06:12:39,135 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-12T06:12:39,135 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-12T06:12:39,135 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-12T06:12:39,135 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-12T06:12:39,136 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-12T06:12:39,136 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-12T06:12:39,136 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-12T06:12:39,136 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-12T06:12:39,137 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-12T06:12:39,138 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-12T06:12:39,138 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-12T06:12:39,138 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-12T06:12:39,138 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,138 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,138 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,138 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,138 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,138 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,138 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,139 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-12T06:12:39,139 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-12T06:12:39,139 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,139 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-12T06:12:39,139 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-12T06:12:39,139 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-12T06:12:39,139 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=60c948f12e02b6ab0066426b6aceea80, REOPEN/MOVE 2024-12-12T06:12:39,140 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true 2024-12-12T06:12:39,140 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:39,140 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:39,140 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-12T06:12:39,140 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:39,140 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:39,140 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:39,140 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN 2024-12-12T06:12:39,140 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN 2024-12-12T06:12:39,140 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN 2024-12-12T06:12:39,141 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8545c1fe97252abe3447dcfaebd5cfa5, ASSIGN 2024-12-12T06:12:39,141 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1202ee0cad3e428893fcdbc630d4f9b9, ASSIGN 2024-12-12T06:12:39,141 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae428798a6433b4d48a205ab3d14ba42, ASSIGN 2024-12-12T06:12:39,141 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4584d26a35c36968de829b34ac08fb5d, ASSIGN 2024-12-12T06:12:39,141 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,141 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,141 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,141 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,141 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,141 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,141 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,141 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,141 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN 2024-12-12T06:12:39,141 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,141 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN 2024-12-12T06:12:39,141 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,141 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN 2024-12-12T06:12:39,141 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8545c1fe97252abe3447dcfaebd5cfa5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,141 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8545c1fe97252abe3447dcfaebd5cfa5, ASSIGN 2024-12-12T06:12:39,141 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1202ee0cad3e428893fcdbc630d4f9b9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,141 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1202ee0cad3e428893fcdbc630d4f9b9, ASSIGN 2024-12-12T06:12:39,141 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae428798a6433b4d48a205ab3d14ba42, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,141 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae428798a6433b4d48a205ab3d14ba42, ASSIGN 2024-12-12T06:12:39,141 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4584d26a35c36968de829b34ac08fb5d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,141 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4584d26a35c36968de829b34ac08fb5d, ASSIGN 2024-12-12T06:12:39,141 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-12T06:12:39,142 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=65 2024-12-12T06:12:39,142 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ffbfd3107920,42275,1733983951224, table=hbase:meta, region=1588230740 2024-12-12T06:12:39,142 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ffbfd3107920,42275,1733983951224, state=OPEN 2024-12-12T06:12:39,147 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45209 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-12T06:12:39,147 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45209 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-12T06:12:39,151 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42275-0x10018b511340003, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:39,151 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46199-0x10018b511340001, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:39,151 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45209-0x10018b511340000, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:39,151 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:39,151 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:39,151 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:39,152 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN to state=OFFLINE, location=null, table=test, region=145c6ba0291376a96e3a37f6b274f63b to restore RIT 2024-12-12T06:12:39,152 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN to state=OFFLINE, location=null, table=test, region=57995ad3e527ada56b846564833341d1 to restore RIT 2024-12-12T06:12:39,152 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN to state=OFFLINE, location=null, table=test, region=5d34bdb8a1414fb25a475fb97fc440b4 to restore RIT 2024-12-12T06:12:39,152 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8545c1fe97252abe3447dcfaebd5cfa5, ASSIGN to state=OFFLINE, location=null, table=test, region=8545c1fe97252abe3447dcfaebd5cfa5 to restore RIT 2024-12-12T06:12:39,152 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1202ee0cad3e428893fcdbc630d4f9b9, ASSIGN to state=OFFLINE, location=null, table=test, region=1202ee0cad3e428893fcdbc630d4f9b9 to restore RIT 2024-12-12T06:12:39,152 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae428798a6433b4d48a205ab3d14ba42, ASSIGN to state=OFFLINE, location=null, table=test, region=ae428798a6433b4d48a205ab3d14ba42 to restore RIT 2024-12-12T06:12:39,152 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4584d26a35c36968de829b34ac08fb5d, ASSIGN to state=OFFLINE, location=null, table=test, region=4584d26a35c36968de829b34ac08fb5d to restore RIT 2024-12-12T06:12:39,152 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733983989152 2024-12-12T06:12:39,153 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-12T06:12:39,153 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-12T06:12:39,154 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-12T06:12:39,155 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-12T06:12:39,155 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae428798a6433b4d48a205ab3d14ba42, ASSIGN 2024-12-12T06:12:39,155 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-12T06:12:39,155 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae428798a6433b4d48a205ab3d14ba42, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-12T06:12:39,157 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8545c1fe97252abe3447dcfaebd5cfa5, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,157 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16838c5df0ab4e3938ad84e9a080def9, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,157 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2eeaafb9a1964f4c6899fa5dde05a463, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,157 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c98bb859cc9e047b211ddb4c5b967ce5, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,157 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b935ee7eeea02b161c0ebc0d05527e91, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,157 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4584d26a35c36968de829b34ac08fb5d, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,157 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b717f65da42dacb67935407f094e8467, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,157 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f2b6a6b11f14670aca9dccceb20d9630, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,157 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57995ad3e527ada56b846564833341d1, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,158 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c77db308a7c1c57fe9ef945e17e28127, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,158 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=782c52424de7d71cb43b8fbfe8e89f7f, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,158 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d34bdb8a1414fb25a475fb97fc440b4, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,158 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d86220c39cb75f305e7ed4d4c9f1191c, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,158 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be4cb5da32eefc543e09e80b426b6938, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,158 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4bf848f8297d89de7b203fcc6678eb5, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,158 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1202ee0cad3e428893fcdbc630d4f9b9, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,158 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=653572572648db72daea94dd65cfb7f8, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,158 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5fe5dc5eb9193cba5339ee508dff867, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,158 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=145c6ba0291376a96e3a37f6b274f63b, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,158 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5162d3c0b3255050cbeed27e4456e5c2, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,158 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d3ce5f64ef1889a3856db9d8c0f63a72, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60c948f12e02b6ab0066426b6aceea80, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=5 2024-12-12T06:12:39,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=764fc3b5431e53dfb445323d0bfd39a6, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=269d56dd3a6671f5c9a30401b73ade89, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae428798a6433b4d48a205ab3d14ba42, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=269eed2924060312949b51c10150f409, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,159 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-12T06:12:39,159 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733984019159 2024-12-12T06:12:39,159 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733984079159 2024-12-12T06:12:39,159 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-12-12T06:12:39,159 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-12T06:12:39,159 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-12T06:12:39,159 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-12T06:12:39,159 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-12T06:12:39,160 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-12T06:12:39,160 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-12T06:12:39,160 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-12T06:12:39,160 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-12T06:12:39,160 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-12T06:12:39,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-12T06:12:39,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-12T06:12:39,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-12T06:12:39,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-12T06:12:39,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-12T06:12:39,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-12T06:12:39,162 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-12T06:12:39,163 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-12T06:12:39,163 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=60c948f12e02b6ab0066426b6aceea80, REOPEN/MOVE 2024-12-12T06:12:39,163 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true 2024-12-12T06:12:39,164 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:39,164 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:39,164 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-12T06:12:39,164 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:39,164 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:39,164 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:39,164 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN 2024-12-12T06:12:39,164 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN 2024-12-12T06:12:39,164 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN 2024-12-12T06:12:39,164 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8545c1fe97252abe3447dcfaebd5cfa5, ASSIGN 2024-12-12T06:12:39,164 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1202ee0cad3e428893fcdbc630d4f9b9, ASSIGN 2024-12-12T06:12:39,164 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae428798a6433b4d48a205ab3d14ba42, ASSIGN 2024-12-12T06:12:39,164 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,164 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,164 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,164 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,164 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,164 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,164 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,164 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,164 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN 2024-12-12T06:12:39,164 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,164 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN 2024-12-12T06:12:39,164 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,164 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN 2024-12-12T06:12:39,164 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8545c1fe97252abe3447dcfaebd5cfa5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,164 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8545c1fe97252abe3447dcfaebd5cfa5, ASSIGN 2024-12-12T06:12:39,164 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1202ee0cad3e428893fcdbc630d4f9b9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,164 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1202ee0cad3e428893fcdbc630d4f9b9, ASSIGN 2024-12-12T06:12:39,164 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae428798a6433b4d48a205ab3d14ba42, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,165 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae428798a6433b4d48a205ab3d14ba42, ASSIGN 2024-12-12T06:12:39,165 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-12T06:12:39,165 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=65 2024-12-12T06:12:39,166 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ffbfd3107920,42275,1733983951224, table=hbase:meta, region=1588230740 2024-12-12T06:12:39,166 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ffbfd3107920,42275,1733983951224, state=OPEN 2024-12-12T06:12:39,176 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46199-0x10018b511340001, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:39,176 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42275-0x10018b511340003, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:39,176 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45209-0x10018b511340000, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:39,176 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:39,176 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:39,176 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:39,177 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN to state=OFFLINE, location=null, table=test, region=145c6ba0291376a96e3a37f6b274f63b to restore RIT 2024-12-12T06:12:39,177 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN to state=OFFLINE, location=null, table=test, region=57995ad3e527ada56b846564833341d1 to restore RIT 2024-12-12T06:12:39,177 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN to state=OFFLINE, location=null, table=test, region=5d34bdb8a1414fb25a475fb97fc440b4 to restore RIT 2024-12-12T06:12:39,178 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8545c1fe97252abe3447dcfaebd5cfa5, ASSIGN to state=OFFLINE, location=null, table=test, region=8545c1fe97252abe3447dcfaebd5cfa5 to restore RIT 2024-12-12T06:12:39,178 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1202ee0cad3e428893fcdbc630d4f9b9, ASSIGN to state=OFFLINE, location=null, table=test, region=1202ee0cad3e428893fcdbc630d4f9b9 to restore RIT 2024-12-12T06:12:39,178 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ae428798a6433b4d48a205ab3d14ba42, ASSIGN to state=OFFLINE, location=null, table=test, region=ae428798a6433b4d48a205ab3d14ba42 to restore RIT 2024-12-12T06:12:39,178 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733983989178 2024-12-12T06:12:39,178 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-12T06:12:39,178 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-12T06:12:39,179 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-12T06:12:39,180 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-12T06:12:39,180 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1202ee0cad3e428893fcdbc630d4f9b9, ASSIGN 2024-12-12T06:12:39,180 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-12T06:12:39,180 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1202ee0cad3e428893fcdbc630d4f9b9, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-12T06:12:39,183 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8545c1fe97252abe3447dcfaebd5cfa5, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,183 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16838c5df0ab4e3938ad84e9a080def9, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,183 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2eeaafb9a1964f4c6899fa5dde05a463, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,183 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c98bb859cc9e047b211ddb4c5b967ce5, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,183 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b935ee7eeea02b161c0ebc0d05527e91, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,183 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4584d26a35c36968de829b34ac08fb5d, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,184 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b717f65da42dacb67935407f094e8467, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,184 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f2b6a6b11f14670aca9dccceb20d9630, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,184 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57995ad3e527ada56b846564833341d1, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,184 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c77db308a7c1c57fe9ef945e17e28127, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,184 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=782c52424de7d71cb43b8fbfe8e89f7f, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,184 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d34bdb8a1414fb25a475fb97fc440b4, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,184 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d86220c39cb75f305e7ed4d4c9f1191c, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,184 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be4cb5da32eefc543e09e80b426b6938, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,185 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4bf848f8297d89de7b203fcc6678eb5, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,185 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1202ee0cad3e428893fcdbc630d4f9b9, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,185 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=653572572648db72daea94dd65cfb7f8, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,185 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5fe5dc5eb9193cba5339ee508dff867, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,185 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=145c6ba0291376a96e3a37f6b274f63b, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,185 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5162d3c0b3255050cbeed27e4456e5c2, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,185 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d3ce5f64ef1889a3856db9d8c0f63a72, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,185 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60c948f12e02b6ab0066426b6aceea80, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=5 2024-12-12T06:12:39,186 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=764fc3b5431e53dfb445323d0bfd39a6, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,186 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=269d56dd3a6671f5c9a30401b73ade89, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,186 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae428798a6433b4d48a205ab3d14ba42, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,186 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=269eed2924060312949b51c10150f409, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,186 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-12T06:12:39,186 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733984019186 2024-12-12T06:12:39,186 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733984079186 2024-12-12T06:12:39,186 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-12-12T06:12:39,186 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-12T06:12:39,186 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-12T06:12:39,186 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-12T06:12:39,186 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-12T06:12:39,187 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-12T06:12:39,187 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-12T06:12:39,187 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-12T06:12:39,187 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-12T06:12:39,188 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-12T06:12:39,190 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-12T06:12:39,190 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-12T06:12:39,190 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-12T06:12:39,190 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,190 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,190 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,190 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,190 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,190 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-12T06:12:39,190 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-12T06:12:39,190 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,190 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-12T06:12:39,191 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-12T06:12:39,191 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-12T06:12:39,191 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=60c948f12e02b6ab0066426b6aceea80, REOPEN/MOVE 2024-12-12T06:12:39,192 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true 2024-12-12T06:12:39,192 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:39,193 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:39,193 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-12T06:12:39,193 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:39,193 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:39,193 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:39,193 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN 2024-12-12T06:12:39,193 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN 2024-12-12T06:12:39,193 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN 2024-12-12T06:12:39,193 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8545c1fe97252abe3447dcfaebd5cfa5, ASSIGN 2024-12-12T06:12:39,193 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1202ee0cad3e428893fcdbc630d4f9b9, ASSIGN 2024-12-12T06:12:39,193 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,193 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,193 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,193 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,193 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,193 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,193 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,193 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,193 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN 2024-12-12T06:12:39,193 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,193 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN 2024-12-12T06:12:39,193 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,193 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN 2024-12-12T06:12:39,193 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8545c1fe97252abe3447dcfaebd5cfa5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,193 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8545c1fe97252abe3447dcfaebd5cfa5, ASSIGN 2024-12-12T06:12:39,194 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1202ee0cad3e428893fcdbc630d4f9b9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,194 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1202ee0cad3e428893fcdbc630d4f9b9, ASSIGN 2024-12-12T06:12:39,194 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-12-12T06:12:39,195 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=65 2024-12-12T06:12:39,195 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ffbfd3107920,42275,1733983951224, table=hbase:meta, region=1588230740 2024-12-12T06:12:39,195 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ffbfd3107920,42275,1733983951224, state=OPEN 2024-12-12T06:12:39,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46199-0x10018b511340001, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:39,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42275-0x10018b511340003, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:39,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45209-0x10018b511340000, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:39,206 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:39,206 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:39,206 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:39,206 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN to state=OFFLINE, location=null, table=test, region=145c6ba0291376a96e3a37f6b274f63b to restore RIT 2024-12-12T06:12:39,206 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN to state=OFFLINE, location=null, table=test, region=57995ad3e527ada56b846564833341d1 to restore RIT 2024-12-12T06:12:39,207 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN to state=OFFLINE, location=null, table=test, region=5d34bdb8a1414fb25a475fb97fc440b4 to restore RIT 2024-12-12T06:12:39,207 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8545c1fe97252abe3447dcfaebd5cfa5, ASSIGN to state=OFFLINE, location=null, table=test, region=8545c1fe97252abe3447dcfaebd5cfa5 to restore RIT 2024-12-12T06:12:39,207 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1202ee0cad3e428893fcdbc630d4f9b9, ASSIGN to state=OFFLINE, location=null, table=test, region=1202ee0cad3e428893fcdbc630d4f9b9 to restore RIT 2024-12-12T06:12:39,207 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733983989207 2024-12-12T06:12:39,207 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-12T06:12:39,207 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-12T06:12:39,208 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-12T06:12:39,209 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-12T06:12:39,209 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8545c1fe97252abe3447dcfaebd5cfa5, ASSIGN 2024-12-12T06:12:39,209 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-12T06:12:39,209 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8545c1fe97252abe3447dcfaebd5cfa5, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-12T06:12:39,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8545c1fe97252abe3447dcfaebd5cfa5, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16838c5df0ab4e3938ad84e9a080def9, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2eeaafb9a1964f4c6899fa5dde05a463, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c98bb859cc9e047b211ddb4c5b967ce5, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b935ee7eeea02b161c0ebc0d05527e91, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,213 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4584d26a35c36968de829b34ac08fb5d, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,213 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b717f65da42dacb67935407f094e8467, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,213 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f2b6a6b11f14670aca9dccceb20d9630, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,213 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57995ad3e527ada56b846564833341d1, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,213 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c77db308a7c1c57fe9ef945e17e28127, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,213 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=782c52424de7d71cb43b8fbfe8e89f7f, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,213 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d34bdb8a1414fb25a475fb97fc440b4, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,213 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d86220c39cb75f305e7ed4d4c9f1191c, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,213 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be4cb5da32eefc543e09e80b426b6938, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,214 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4bf848f8297d89de7b203fcc6678eb5, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,214 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1202ee0cad3e428893fcdbc630d4f9b9, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,214 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=653572572648db72daea94dd65cfb7f8, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,214 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5fe5dc5eb9193cba5339ee508dff867, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,214 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=145c6ba0291376a96e3a37f6b274f63b, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,214 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5162d3c0b3255050cbeed27e4456e5c2, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,214 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d3ce5f64ef1889a3856db9d8c0f63a72, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,214 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60c948f12e02b6ab0066426b6aceea80, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=5 2024-12-12T06:12:39,214 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=764fc3b5431e53dfb445323d0bfd39a6, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,214 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=269d56dd3a6671f5c9a30401b73ade89, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,215 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae428798a6433b4d48a205ab3d14ba42, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,215 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=269eed2924060312949b51c10150f409, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,215 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-12T06:12:39,215 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733984019215 2024-12-12T06:12:39,215 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733984079215 2024-12-12T06:12:39,215 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-12-12T06:12:39,215 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-12T06:12:39,215 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-12T06:12:39,215 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-12T06:12:39,215 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-12T06:12:39,215 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-12T06:12:39,215 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-12T06:12:39,215 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-12T06:12:39,216 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-12T06:12:39,216 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-12T06:12:39,218 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-12T06:12:39,218 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-12T06:12:39,218 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-12T06:12:39,218 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,218 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,218 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,219 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,219 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-12T06:12:39,219 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-12T06:12:39,219 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,219 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-12T06:12:39,219 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-12T06:12:39,220 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-12T06:12:39,220 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=60c948f12e02b6ab0066426b6aceea80, REOPEN/MOVE 2024-12-12T06:12:39,221 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true 2024-12-12T06:12:39,221 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:39,221 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:39,221 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-12T06:12:39,221 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:39,221 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:39,221 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:39,221 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN 2024-12-12T06:12:39,221 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN 2024-12-12T06:12:39,221 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN 2024-12-12T06:12:39,221 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8545c1fe97252abe3447dcfaebd5cfa5, ASSIGN 2024-12-12T06:12:39,221 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,221 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,221 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,221 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,221 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,221 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,221 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,221 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,222 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN 2024-12-12T06:12:39,222 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,222 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN 2024-12-12T06:12:39,222 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,222 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN 2024-12-12T06:12:39,222 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8545c1fe97252abe3447dcfaebd5cfa5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,222 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8545c1fe97252abe3447dcfaebd5cfa5, ASSIGN 2024-12-12T06:12:39,222 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-12-12T06:12:39,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=65 2024-12-12T06:12:39,223 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ffbfd3107920,42275,1733983951224, table=hbase:meta, region=1588230740 2024-12-12T06:12:39,223 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ffbfd3107920,42275,1733983951224, state=OPEN 2024-12-12T06:12:39,234 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42275-0x10018b511340003, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:39,234 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45209-0x10018b511340000, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:39,234 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46199-0x10018b511340001, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:39,234 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:39,234 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:39,235 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:39,235 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN to state=OFFLINE, location=null, table=test, region=145c6ba0291376a96e3a37f6b274f63b to restore RIT 2024-12-12T06:12:39,235 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN to state=OFFLINE, location=null, table=test, region=57995ad3e527ada56b846564833341d1 to restore RIT 2024-12-12T06:12:39,235 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN to state=OFFLINE, location=null, table=test, region=5d34bdb8a1414fb25a475fb97fc440b4 to restore RIT 2024-12-12T06:12:39,235 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8545c1fe97252abe3447dcfaebd5cfa5, ASSIGN to state=OFFLINE, location=null, table=test, region=8545c1fe97252abe3447dcfaebd5cfa5 to restore RIT 2024-12-12T06:12:39,235 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733983989235 2024-12-12T06:12:39,235 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-12T06:12:39,236 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-12T06:12:39,237 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-12T06:12:39,238 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-12T06:12:39,238 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN 2024-12-12T06:12:39,238 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-12T06:12:39,238 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-12T06:12:39,241 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8545c1fe97252abe3447dcfaebd5cfa5, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,241 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16838c5df0ab4e3938ad84e9a080def9, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,242 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2eeaafb9a1964f4c6899fa5dde05a463, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,242 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c98bb859cc9e047b211ddb4c5b967ce5, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,242 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b935ee7eeea02b161c0ebc0d05527e91, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,242 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4584d26a35c36968de829b34ac08fb5d, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,242 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b717f65da42dacb67935407f094e8467, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,242 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f2b6a6b11f14670aca9dccceb20d9630, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,242 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57995ad3e527ada56b846564833341d1, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,242 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c77db308a7c1c57fe9ef945e17e28127, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,242 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=782c52424de7d71cb43b8fbfe8e89f7f, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,242 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d34bdb8a1414fb25a475fb97fc440b4, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,242 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d86220c39cb75f305e7ed4d4c9f1191c, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,243 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be4cb5da32eefc543e09e80b426b6938, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,243 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4bf848f8297d89de7b203fcc6678eb5, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,243 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1202ee0cad3e428893fcdbc630d4f9b9, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,243 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=653572572648db72daea94dd65cfb7f8, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,243 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5fe5dc5eb9193cba5339ee508dff867, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,243 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=145c6ba0291376a96e3a37f6b274f63b, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,243 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5162d3c0b3255050cbeed27e4456e5c2, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,243 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d3ce5f64ef1889a3856db9d8c0f63a72, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,243 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60c948f12e02b6ab0066426b6aceea80, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=5 2024-12-12T06:12:39,243 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=764fc3b5431e53dfb445323d0bfd39a6, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,244 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=269d56dd3a6671f5c9a30401b73ade89, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,244 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae428798a6433b4d48a205ab3d14ba42, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,244 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=269eed2924060312949b51c10150f409, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,244 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-12T06:12:39,244 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733984019244 2024-12-12T06:12:39,244 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733984079244 2024-12-12T06:12:39,244 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-12-12T06:12:39,244 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-12T06:12:39,244 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-12T06:12:39,244 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-12T06:12:39,244 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-12T06:12:39,244 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-12T06:12:39,244 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-12T06:12:39,244 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-12T06:12:39,245 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-12T06:12:39,245 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-12T06:12:39,247 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-12T06:12:39,247 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-12T06:12:39,247 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-12T06:12:39,247 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,247 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,247 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,247 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-12T06:12:39,247 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-12T06:12:39,247 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,247 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-12T06:12:39,247 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-12T06:12:39,248 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-12T06:12:39,248 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=60c948f12e02b6ab0066426b6aceea80, REOPEN/MOVE 2024-12-12T06:12:39,249 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true 2024-12-12T06:12:39,249 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:39,249 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:39,249 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-12T06:12:39,249 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:39,249 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:39,249 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:39,249 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN 2024-12-12T06:12:39,249 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN 2024-12-12T06:12:39,249 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN 2024-12-12T06:12:39,249 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,249 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,249 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,249 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,249 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,249 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,249 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,249 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,249 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN 2024-12-12T06:12:39,249 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,250 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN 2024-12-12T06:12:39,250 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,250 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN 2024-12-12T06:12:39,250 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-12T06:12:39,250 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=65 2024-12-12T06:12:39,251 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ffbfd3107920,42275,1733983951224, table=hbase:meta, region=1588230740 2024-12-12T06:12:39,251 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ffbfd3107920,42275,1733983951224, state=OPEN 2024-12-12T06:12:39,253 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45209 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-12T06:12:39,253 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45209 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-12T06:12:39,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42275-0x10018b511340003, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:39,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46199-0x10018b511340001, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:39,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45209-0x10018b511340000, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:39,259 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:39,260 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:39,260 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:39,260 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN to state=OFFLINE, location=null, table=test, region=145c6ba0291376a96e3a37f6b274f63b to restore RIT 2024-12-12T06:12:39,260 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN to state=OFFLINE, location=null, table=test, region=57995ad3e527ada56b846564833341d1 to restore RIT 2024-12-12T06:12:39,260 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5d34bdb8a1414fb25a475fb97fc440b4, ASSIGN to state=OFFLINE, location=null, table=test, region=5d34bdb8a1414fb25a475fb97fc440b4 to restore RIT 2024-12-12T06:12:39,261 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733983989261 2024-12-12T06:12:39,261 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-12T06:12:39,261 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-12T06:12:39,262 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-12T06:12:39,263 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-12T06:12:39,263 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN 2024-12-12T06:12:39,263 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-12T06:12:39,263 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-12T06:12:39,265 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8545c1fe97252abe3447dcfaebd5cfa5, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,265 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16838c5df0ab4e3938ad84e9a080def9, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,265 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2eeaafb9a1964f4c6899fa5dde05a463, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,265 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c98bb859cc9e047b211ddb4c5b967ce5, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,265 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b935ee7eeea02b161c0ebc0d05527e91, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,265 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4584d26a35c36968de829b34ac08fb5d, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,266 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b717f65da42dacb67935407f094e8467, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,266 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f2b6a6b11f14670aca9dccceb20d9630, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,266 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57995ad3e527ada56b846564833341d1, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,266 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c77db308a7c1c57fe9ef945e17e28127, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,266 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=782c52424de7d71cb43b8fbfe8e89f7f, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,266 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d34bdb8a1414fb25a475fb97fc440b4, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,266 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d86220c39cb75f305e7ed4d4c9f1191c, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,266 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be4cb5da32eefc543e09e80b426b6938, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,266 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4bf848f8297d89de7b203fcc6678eb5, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,266 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1202ee0cad3e428893fcdbc630d4f9b9, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,266 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=653572572648db72daea94dd65cfb7f8, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,266 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5fe5dc5eb9193cba5339ee508dff867, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,266 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=145c6ba0291376a96e3a37f6b274f63b, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,267 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5162d3c0b3255050cbeed27e4456e5c2, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,267 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d3ce5f64ef1889a3856db9d8c0f63a72, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,267 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60c948f12e02b6ab0066426b6aceea80, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=5 2024-12-12T06:12:39,267 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=764fc3b5431e53dfb445323d0bfd39a6, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,267 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=269d56dd3a6671f5c9a30401b73ade89, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,267 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae428798a6433b4d48a205ab3d14ba42, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,267 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=269eed2924060312949b51c10150f409, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,267 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-12T06:12:39,267 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733984019267 2024-12-12T06:12:39,267 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733984079267 2024-12-12T06:12:39,267 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-12-12T06:12:39,267 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-12T06:12:39,267 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-12T06:12:39,267 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-12T06:12:39,267 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-12T06:12:39,268 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-12T06:12:39,268 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-12T06:12:39,268 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-12T06:12:39,268 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-12T06:12:39,269 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-12T06:12:39,270 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-12T06:12:39,270 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-12T06:12:39,270 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-12T06:12:39,270 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,270 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,270 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-12T06:12:39,270 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-12T06:12:39,270 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,270 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-12T06:12:39,270 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-12T06:12:39,271 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-12T06:12:39,271 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=60c948f12e02b6ab0066426b6aceea80, REOPEN/MOVE 2024-12-12T06:12:39,272 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true 2024-12-12T06:12:39,272 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:39,272 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:39,272 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-12T06:12:39,272 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:39,272 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:39,272 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:39,272 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN 2024-12-12T06:12:39,272 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN 2024-12-12T06:12:39,272 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,272 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,272 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,272 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,272 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,272 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,272 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,272 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,272 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN 2024-12-12T06:12:39,272 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,272 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN 2024-12-12T06:12:39,272 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-12-12T06:12:39,273 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=65 2024-12-12T06:12:39,273 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ffbfd3107920,42275,1733983951224, table=hbase:meta, region=1588230740 2024-12-12T06:12:39,273 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ffbfd3107920,42275,1733983951224, state=OPEN 2024-12-12T06:12:39,309 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46199-0x10018b511340001, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:39,309 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42275-0x10018b511340003, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:39,309 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45209-0x10018b511340000, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:39,309 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:39,309 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:39,309 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:39,309 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN to state=OFFLINE, location=null, table=test, region=145c6ba0291376a96e3a37f6b274f63b to restore RIT 2024-12-12T06:12:39,310 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57995ad3e527ada56b846564833341d1, ASSIGN to state=OFFLINE, location=null, table=test, region=57995ad3e527ada56b846564833341d1 to restore RIT 2024-12-12T06:12:39,310 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733983989310 2024-12-12T06:12:39,310 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-12T06:12:39,311 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-12T06:12:39,312 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-12T06:12:39,314 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-12T06:12:39,315 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN 2024-12-12T06:12:39,315 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-12T06:12:39,315 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-12T06:12:39,316 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8545c1fe97252abe3447dcfaebd5cfa5, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,316 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16838c5df0ab4e3938ad84e9a080def9, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,316 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2eeaafb9a1964f4c6899fa5dde05a463, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,316 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c98bb859cc9e047b211ddb4c5b967ce5, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,316 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b935ee7eeea02b161c0ebc0d05527e91, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,316 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4584d26a35c36968de829b34ac08fb5d, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,316 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b717f65da42dacb67935407f094e8467, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,316 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f2b6a6b11f14670aca9dccceb20d9630, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,316 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57995ad3e527ada56b846564833341d1, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,316 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c77db308a7c1c57fe9ef945e17e28127, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,316 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=782c52424de7d71cb43b8fbfe8e89f7f, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,316 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d34bdb8a1414fb25a475fb97fc440b4, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,317 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d86220c39cb75f305e7ed4d4c9f1191c, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,317 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be4cb5da32eefc543e09e80b426b6938, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,317 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4bf848f8297d89de7b203fcc6678eb5, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,317 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1202ee0cad3e428893fcdbc630d4f9b9, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,317 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=653572572648db72daea94dd65cfb7f8, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,317 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5fe5dc5eb9193cba5339ee508dff867, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,317 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=145c6ba0291376a96e3a37f6b274f63b, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,317 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5162d3c0b3255050cbeed27e4456e5c2, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,317 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d3ce5f64ef1889a3856db9d8c0f63a72, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,317 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60c948f12e02b6ab0066426b6aceea80, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=5 2024-12-12T06:12:39,317 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=764fc3b5431e53dfb445323d0bfd39a6, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,317 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=269d56dd3a6671f5c9a30401b73ade89, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,318 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae428798a6433b4d48a205ab3d14ba42, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,318 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=269eed2924060312949b51c10150f409, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,318 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-12T06:12:39,318 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733984019318 2024-12-12T06:12:39,318 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733984079318 2024-12-12T06:12:39,318 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-12-12T06:12:39,318 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-12T06:12:39,318 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-12T06:12:39,318 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-12T06:12:39,318 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-12T06:12:39,318 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-12T06:12:39,318 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-12T06:12:39,318 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-12T06:12:39,318 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-12T06:12:39,319 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-12T06:12:39,321 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-12T06:12:39,321 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-12T06:12:39,321 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-12T06:12:39,321 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,321 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-12T06:12:39,321 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-12T06:12:39,321 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,322 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-12T06:12:39,322 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-12T06:12:39,323 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-12T06:12:39,323 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=60c948f12e02b6ab0066426b6aceea80, REOPEN/MOVE 2024-12-12T06:12:39,324 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true 2024-12-12T06:12:39,324 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:39,324 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:39,324 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-12T06:12:39,324 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:39,324 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:39,324 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:39,324 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN 2024-12-12T06:12:39,325 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,325 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,325 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,325 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,325 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,325 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,325 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,325 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,325 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN 2024-12-12T06:12:39,325 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-12-12T06:12:39,326 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=65 2024-12-12T06:12:39,326 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ffbfd3107920,42275,1733983951224, table=hbase:meta, region=1588230740 2024-12-12T06:12:39,327 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ffbfd3107920,42275,1733983951224, state=OPEN 2024-12-12T06:12:39,343 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46199-0x10018b511340001, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:39,343 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42275-0x10018b511340003, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:39,343 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45209-0x10018b511340000, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:39,343 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:39,343 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:39,343 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:39,344 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=145c6ba0291376a96e3a37f6b274f63b, ASSIGN to state=OFFLINE, location=null, table=test, region=145c6ba0291376a96e3a37f6b274f63b to restore RIT 2024-12-12T06:12:39,344 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733983989344 2024-12-12T06:12:39,344 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-12T06:12:39,344 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-12T06:12:39,345 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-12T06:12:39,346 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-12T06:12:39,346 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:39,346 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-12T06:12:39,346 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-12T06:12:39,350 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8545c1fe97252abe3447dcfaebd5cfa5, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,350 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16838c5df0ab4e3938ad84e9a080def9, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,350 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2eeaafb9a1964f4c6899fa5dde05a463, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,350 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c98bb859cc9e047b211ddb4c5b967ce5, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,350 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b935ee7eeea02b161c0ebc0d05527e91, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,350 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4584d26a35c36968de829b34ac08fb5d, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,350 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b717f65da42dacb67935407f094e8467, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,351 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f2b6a6b11f14670aca9dccceb20d9630, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,351 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57995ad3e527ada56b846564833341d1, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,351 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c77db308a7c1c57fe9ef945e17e28127, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,351 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=782c52424de7d71cb43b8fbfe8e89f7f, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,351 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d34bdb8a1414fb25a475fb97fc440b4, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,351 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d86220c39cb75f305e7ed4d4c9f1191c, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,351 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be4cb5da32eefc543e09e80b426b6938, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,351 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4bf848f8297d89de7b203fcc6678eb5, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,351 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1202ee0cad3e428893fcdbc630d4f9b9, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,351 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=653572572648db72daea94dd65cfb7f8, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,351 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5fe5dc5eb9193cba5339ee508dff867, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,351 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=145c6ba0291376a96e3a37f6b274f63b, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,351 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5162d3c0b3255050cbeed27e4456e5c2, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,352 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d3ce5f64ef1889a3856db9d8c0f63a72, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,352 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60c948f12e02b6ab0066426b6aceea80, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=5 2024-12-12T06:12:39,352 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=764fc3b5431e53dfb445323d0bfd39a6, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,352 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=269d56dd3a6671f5c9a30401b73ade89, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,352 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae428798a6433b4d48a205ab3d14ba42, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,352 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=269eed2924060312949b51c10150f409, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,352 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-12T06:12:39,352 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733984019352 2024-12-12T06:12:39,352 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733984079352 2024-12-12T06:12:39,352 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-12-12T06:12:39,352 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-12T06:12:39,352 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-12T06:12:39,352 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-12T06:12:39,352 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-12T06:12:39,353 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-12T06:12:39,353 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-12T06:12:39,353 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-12T06:12:39,353 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-12T06:12:39,354 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-12T06:12:39,356 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-12T06:12:39,356 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-12T06:12:39,356 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-12T06:12:39,356 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-12T06:12:39,356 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-12T06:12:39,356 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,356 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-12T06:12:39,356 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-12T06:12:39,357 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-12T06:12:39,357 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=60c948f12e02b6ab0066426b6aceea80, REOPEN/MOVE 2024-12-12T06:12:39,358 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true 2024-12-12T06:12:39,358 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:39,358 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:39,358 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-12T06:12:39,359 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:39,359 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:39,359 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:39,359 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,359 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,359 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,359 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,359 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,359 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,359 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,359 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-12-12T06:12:39,360 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=65 2024-12-12T06:12:39,360 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ffbfd3107920,42275,1733983951224, table=hbase:meta, region=1588230740 2024-12-12T06:12:39,361 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ffbfd3107920,42275,1733983951224, state=OPEN 2024-12-12T06:12:39,361 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45209 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-12T06:12:39,362 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45209 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-12T06:12:39,372 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46199-0x10018b511340001, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:39,372 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42275-0x10018b511340003, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:39,372 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45209-0x10018b511340000, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:39,373 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:39,373 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:39,373 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:39,373 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733983989373 2024-12-12T06:12:39,374 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-12T06:12:39,374 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-12T06:12:39,376 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-12T06:12:39,377 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-12T06:12:39,377 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:39,377 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-12T06:12:39,377 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-12T06:12:39,380 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8545c1fe97252abe3447dcfaebd5cfa5, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,380 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16838c5df0ab4e3938ad84e9a080def9, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,380 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2eeaafb9a1964f4c6899fa5dde05a463, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,380 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c98bb859cc9e047b211ddb4c5b967ce5, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,380 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b935ee7eeea02b161c0ebc0d05527e91, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,381 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4584d26a35c36968de829b34ac08fb5d, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,381 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b717f65da42dacb67935407f094e8467, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,381 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f2b6a6b11f14670aca9dccceb20d9630, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,381 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57995ad3e527ada56b846564833341d1, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,381 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c77db308a7c1c57fe9ef945e17e28127, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,381 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=782c52424de7d71cb43b8fbfe8e89f7f, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,381 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d34bdb8a1414fb25a475fb97fc440b4, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,381 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d86220c39cb75f305e7ed4d4c9f1191c, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,381 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be4cb5da32eefc543e09e80b426b6938, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,382 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4bf848f8297d89de7b203fcc6678eb5, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,382 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1202ee0cad3e428893fcdbc630d4f9b9, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,382 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=653572572648db72daea94dd65cfb7f8, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,382 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5fe5dc5eb9193cba5339ee508dff867, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,382 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=145c6ba0291376a96e3a37f6b274f63b, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,382 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5162d3c0b3255050cbeed27e4456e5c2, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,382 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d3ce5f64ef1889a3856db9d8c0f63a72, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,382 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60c948f12e02b6ab0066426b6aceea80, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=5 2024-12-12T06:12:39,382 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=764fc3b5431e53dfb445323d0bfd39a6, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,382 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=269d56dd3a6671f5c9a30401b73ade89, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,383 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae428798a6433b4d48a205ab3d14ba42, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,383 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=269eed2924060312949b51c10150f409, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,383 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-12T06:12:39,383 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733984019383 2024-12-12T06:12:39,383 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733984079383 2024-12-12T06:12:39,383 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-12-12T06:12:39,383 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-12T06:12:39,383 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-12T06:12:39,383 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-12T06:12:39,383 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-12T06:12:39,384 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-12T06:12:39,384 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-12T06:12:39,384 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-12T06:12:39,384 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-12T06:12:39,385 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-12T06:12:39,387 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-12T06:12:39,387 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-12T06:12:39,387 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-12T06:12:39,387 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-12T06:12:39,387 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,387 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-12T06:12:39,387 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-12T06:12:39,388 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-12T06:12:39,388 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=60c948f12e02b6ab0066426b6aceea80, REOPEN/MOVE 2024-12-12T06:12:39,389 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true 2024-12-12T06:12:39,389 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:39,389 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:39,389 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-12T06:12:39,389 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:39,389 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 2024-12-12T06:12:39,389 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,389 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,389 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,389 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,389 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,389 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.1733983952728, worker=ffbfd3107920,46199,1733983951048 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,389 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-12T06:12:39,390 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=65 2024-12-12T06:12:39,390 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ffbfd3107920,42275,1733983951224, table=hbase:meta, region=1588230740 2024-12-12T06:12:39,391 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ffbfd3107920,42275,1733983951224, state=OPEN 2024-12-12T06:12:39,401 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42275-0x10018b511340003, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:39,401 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45209-0x10018b511340000, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:39,401 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46199-0x10018b511340001, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:39,401 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:39,401 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:39,401 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:39,403 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733983989402 2024-12-12T06:12:39,403 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-12T06:12:39,403 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-12T06:12:39,404 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-12T06:12:39,407 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-12T06:12:39,407 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:39,407 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-12T06:12:39,407 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ffbfd3107920,42275,1733983951224 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-12T06:12:39,408 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8545c1fe97252abe3447dcfaebd5cfa5, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,408 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16838c5df0ab4e3938ad84e9a080def9, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,408 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2eeaafb9a1964f4c6899fa5dde05a463, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,408 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c98bb859cc9e047b211ddb4c5b967ce5, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,408 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b935ee7eeea02b161c0ebc0d05527e91, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,409 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4584d26a35c36968de829b34ac08fb5d, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,409 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b717f65da42dacb67935407f094e8467, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,409 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f2b6a6b11f14670aca9dccceb20d9630, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,409 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57995ad3e527ada56b846564833341d1, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,409 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c77db308a7c1c57fe9ef945e17e28127, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,409 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=782c52424de7d71cb43b8fbfe8e89f7f, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,409 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d34bdb8a1414fb25a475fb97fc440b4, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,409 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d86220c39cb75f305e7ed4d4c9f1191c, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,409 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be4cb5da32eefc543e09e80b426b6938, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,410 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4bf848f8297d89de7b203fcc6678eb5, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,410 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1202ee0cad3e428893fcdbc630d4f9b9, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,410 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=653572572648db72daea94dd65cfb7f8, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,410 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5fe5dc5eb9193cba5339ee508dff867, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,410 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=145c6ba0291376a96e3a37f6b274f63b, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,410 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5162d3c0b3255050cbeed27e4456e5c2, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,410 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d3ce5f64ef1889a3856db9d8c0f63a72, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,410 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60c948f12e02b6ab0066426b6aceea80, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=5 2024-12-12T06:12:39,410 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=764fc3b5431e53dfb445323d0bfd39a6, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,410 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=269d56dd3a6671f5c9a30401b73ade89, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,410 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae428798a6433b4d48a205ab3d14ba42, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,411 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=269eed2924060312949b51c10150f409, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,411 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-12T06:12:39,411 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733984019411 2024-12-12T06:12:39,411 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733984079411 2024-12-12T06:12:39,411 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-12-12T06:12:39,411 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-12T06:12:39,411 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-12T06:12:39,411 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-12T06:12:39,411 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-12T06:12:39,411 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-12T06:12:39,411 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-12T06:12:39,412 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-12T06:12:39,412 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-12T06:12:39,413 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-12T06:12:39,415 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-12T06:12:39,415 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-12T06:12:39,415 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-12T06:12:39,415 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,415 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-12T06:12:39,416 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-12T06:12:39,416 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-12T06:12:39,417 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=60c948f12e02b6ab0066426b6aceea80, REOPEN/MOVE 2024-12-12T06:12:39,417 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true 2024-12-12T06:12:39,417 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:39,418 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:39,418 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-12T06:12:39,418 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:39,418 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,418 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,418 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,418 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,418 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,418 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-12-12T06:12:39,419 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=65 2024-12-12T06:12:39,419 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ffbfd3107920,42275,1733983951224, table=hbase:meta, region=1588230740 2024-12-12T06:12:39,419 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ffbfd3107920,42275,1733983951224, state=OPEN 2024-12-12T06:12:39,431 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45209-0x10018b511340000, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:39,431 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46199-0x10018b511340001, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:39,431 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42275-0x10018b511340003, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:39,431 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:39,431 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:39,431 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:39,432 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733983989432 2024-12-12T06:12:39,432 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-12T06:12:39,432 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-12T06:12:39,433 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-12T06:12:39,434 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-12T06:12:39,434 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-12T06:12:39,434 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-12T06:12:39,435 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-12T06:12:39,437 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8545c1fe97252abe3447dcfaebd5cfa5, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,438 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16838c5df0ab4e3938ad84e9a080def9, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,438 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2eeaafb9a1964f4c6899fa5dde05a463, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,438 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c98bb859cc9e047b211ddb4c5b967ce5, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,438 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b935ee7eeea02b161c0ebc0d05527e91, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,438 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4584d26a35c36968de829b34ac08fb5d, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,438 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b717f65da42dacb67935407f094e8467, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,438 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f2b6a6b11f14670aca9dccceb20d9630, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,438 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57995ad3e527ada56b846564833341d1, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,438 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c77db308a7c1c57fe9ef945e17e28127, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,439 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=782c52424de7d71cb43b8fbfe8e89f7f, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,439 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d34bdb8a1414fb25a475fb97fc440b4, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,439 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d86220c39cb75f305e7ed4d4c9f1191c, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,439 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be4cb5da32eefc543e09e80b426b6938, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,439 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4bf848f8297d89de7b203fcc6678eb5, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,439 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1202ee0cad3e428893fcdbc630d4f9b9, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,439 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=653572572648db72daea94dd65cfb7f8, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,439 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5fe5dc5eb9193cba5339ee508dff867, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,439 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=145c6ba0291376a96e3a37f6b274f63b, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,439 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5162d3c0b3255050cbeed27e4456e5c2, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,440 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d3ce5f64ef1889a3856db9d8c0f63a72, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,440 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60c948f12e02b6ab0066426b6aceea80, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=5 2024-12-12T06:12:39,440 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=764fc3b5431e53dfb445323d0bfd39a6, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,440 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=269d56dd3a6671f5c9a30401b73ade89, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,440 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae428798a6433b4d48a205ab3d14ba42, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,440 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=269eed2924060312949b51c10150f409, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,440 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-12T06:12:39,440 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733984019440 2024-12-12T06:12:39,440 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733984079440 2024-12-12T06:12:39,440 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-12-12T06:12:39,441 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-12T06:12:39,441 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-12T06:12:39,441 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-12T06:12:39,441 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-12T06:12:39,441 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-12T06:12:39,441 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-12T06:12:39,441 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-12T06:12:39,441 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-12T06:12:39,442 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-12T06:12:39,444 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-12T06:12:39,444 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-12T06:12:39,444 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-12T06:12:39,444 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-12T06:12:39,444 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-12T06:12:39,445 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-12T06:12:39,445 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=60c948f12e02b6ab0066426b6aceea80, REOPEN/MOVE 2024-12-12T06:12:39,446 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true 2024-12-12T06:12:39,446 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:39,446 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:39,446 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-12T06:12:39,446 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,446 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,446 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,446 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,446 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-12T06:12:39,447 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=65 2024-12-12T06:12:39,447 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ffbfd3107920,42275,1733983951224, table=hbase:meta, region=1588230740 2024-12-12T06:12:39,448 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ffbfd3107920,42275,1733983951224, state=OPEN 2024-12-12T06:12:39,456 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45209-0x10018b511340000, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:39,456 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46199-0x10018b511340001, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:39,456 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42275-0x10018b511340003, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:39,456 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:39,456 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:39,456 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:39,457 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733983989457 2024-12-12T06:12:39,457 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-12T06:12:39,457 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-12T06:12:39,458 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-12T06:12:39,459 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-12T06:12:39,459 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:39,459 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-12T06:12:39,459 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-12T06:12:39,462 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8545c1fe97252abe3447dcfaebd5cfa5, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,462 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16838c5df0ab4e3938ad84e9a080def9, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,462 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2eeaafb9a1964f4c6899fa5dde05a463, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,462 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c98bb859cc9e047b211ddb4c5b967ce5, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,462 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b935ee7eeea02b161c0ebc0d05527e91, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,462 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4584d26a35c36968de829b34ac08fb5d, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,463 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b717f65da42dacb67935407f094e8467, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,463 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f2b6a6b11f14670aca9dccceb20d9630, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,463 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57995ad3e527ada56b846564833341d1, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,463 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c77db308a7c1c57fe9ef945e17e28127, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,463 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=782c52424de7d71cb43b8fbfe8e89f7f, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,463 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d34bdb8a1414fb25a475fb97fc440b4, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,463 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d86220c39cb75f305e7ed4d4c9f1191c, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,463 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be4cb5da32eefc543e09e80b426b6938, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,463 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4bf848f8297d89de7b203fcc6678eb5, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,464 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1202ee0cad3e428893fcdbc630d4f9b9, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,464 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=653572572648db72daea94dd65cfb7f8, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,464 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5fe5dc5eb9193cba5339ee508dff867, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,464 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=145c6ba0291376a96e3a37f6b274f63b, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,464 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5162d3c0b3255050cbeed27e4456e5c2, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,464 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d3ce5f64ef1889a3856db9d8c0f63a72, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,464 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60c948f12e02b6ab0066426b6aceea80, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=5 2024-12-12T06:12:39,464 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=764fc3b5431e53dfb445323d0bfd39a6, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,464 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=269d56dd3a6671f5c9a30401b73ade89, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,464 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae428798a6433b4d48a205ab3d14ba42, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,465 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=269eed2924060312949b51c10150f409, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,465 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-12T06:12:39,465 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733984019465 2024-12-12T06:12:39,465 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733984079465 2024-12-12T06:12:39,465 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-12-12T06:12:39,465 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-12T06:12:39,465 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-12T06:12:39,465 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-12T06:12:39,465 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-12T06:12:39,465 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-12T06:12:39,466 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-12T06:12:39,466 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-12T06:12:39,466 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-12T06:12:39,467 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-12T06:12:39,469 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-12T06:12:39,469 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-12T06:12:39,469 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-12T06:12:39,469 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-12T06:12:39,470 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-12T06:12:39,470 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=60c948f12e02b6ab0066426b6aceea80, REOPEN/MOVE 2024-12-12T06:12:39,471 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true 2024-12-12T06:12:39,471 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:39,471 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:39,471 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,471 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,471 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,472 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-12T06:12:39,473 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=65 2024-12-12T06:12:39,473 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ffbfd3107920,42275,1733983951224, table=hbase:meta, region=1588230740 2024-12-12T06:12:39,473 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ffbfd3107920,42275,1733983951224, state=OPEN 2024-12-12T06:12:39,475 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45209 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-12T06:12:39,476 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45209 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-12T06:12:39,484 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42275-0x10018b511340003, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:39,484 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46199-0x10018b511340001, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:39,484 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45209-0x10018b511340000, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:39,484 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:39,484 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:39,485 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:39,485 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733983989485 2024-12-12T06:12:39,485 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-12T06:12:39,485 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-12T06:12:39,486 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-12T06:12:39,487 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-12T06:12:39,487 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:39,487 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-12T06:12:39,487 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-12T06:12:39,490 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8545c1fe97252abe3447dcfaebd5cfa5, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,490 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16838c5df0ab4e3938ad84e9a080def9, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,490 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2eeaafb9a1964f4c6899fa5dde05a463, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,490 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c98bb859cc9e047b211ddb4c5b967ce5, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,490 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b935ee7eeea02b161c0ebc0d05527e91, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,491 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4584d26a35c36968de829b34ac08fb5d, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,491 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b717f65da42dacb67935407f094e8467, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,491 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f2b6a6b11f14670aca9dccceb20d9630, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,491 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57995ad3e527ada56b846564833341d1, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,491 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c77db308a7c1c57fe9ef945e17e28127, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,491 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=782c52424de7d71cb43b8fbfe8e89f7f, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,491 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d34bdb8a1414fb25a475fb97fc440b4, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,491 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d86220c39cb75f305e7ed4d4c9f1191c, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,491 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be4cb5da32eefc543e09e80b426b6938, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,491 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4bf848f8297d89de7b203fcc6678eb5, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,491 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1202ee0cad3e428893fcdbc630d4f9b9, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,491 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=653572572648db72daea94dd65cfb7f8, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,492 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5fe5dc5eb9193cba5339ee508dff867, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,492 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=145c6ba0291376a96e3a37f6b274f63b, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,492 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5162d3c0b3255050cbeed27e4456e5c2, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,492 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d3ce5f64ef1889a3856db9d8c0f63a72, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,492 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60c948f12e02b6ab0066426b6aceea80, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=5 2024-12-12T06:12:39,492 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=764fc3b5431e53dfb445323d0bfd39a6, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,492 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=269d56dd3a6671f5c9a30401b73ade89, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,492 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae428798a6433b4d48a205ab3d14ba42, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,492 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=269eed2924060312949b51c10150f409, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,492 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-12T06:12:39,492 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733984019492 2024-12-12T06:12:39,492 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733984079492 2024-12-12T06:12:39,492 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-12-12T06:12:39,492 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-12T06:12:39,492 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-12T06:12:39,492 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-12T06:12:39,492 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-12T06:12:39,493 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-12T06:12:39,493 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-12T06:12:39,493 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-12T06:12:39,493 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-12T06:12:39,494 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-12T06:12:39,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-12T06:12:39,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-12T06:12:39,495 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-12T06:12:39,495 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-12T06:12:39,495 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=60c948f12e02b6ab0066426b6aceea80, REOPEN/MOVE 2024-12-12T06:12:39,496 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true 2024-12-12T06:12:39,496 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 2024-12-12T06:12:39,496 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,496 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure ffbfd3107920%2C33455%2C1733983951151.meta.1733983953175.meta, worker=ffbfd3107920,42275,1733983951224 didn't hold the lock before restarting, skip acquiring lock. 2024-12-12T06:12:39,496 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-12-12T06:12:39,497 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=65 2024-12-12T06:12:39,497 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ffbfd3107920,42275,1733983951224, table=hbase:meta, region=1588230740 2024-12-12T06:12:39,497 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ffbfd3107920,42275,1733983951224, state=OPEN 2024-12-12T06:12:39,506 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45209-0x10018b511340000, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:39,506 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46199-0x10018b511340001, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:39,506 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42275-0x10018b511340003, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:39,506 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:39,506 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:39,506 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:39,507 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733983989507 2024-12-12T06:12:39,507 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-12T06:12:39,507 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-12T06:12:39,508 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-12T06:12:39,509 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-12T06:12:39,509 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true 2024-12-12T06:12:39,509 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-12T06:12:39,510 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1604) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-12T06:12:39,513 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8545c1fe97252abe3447dcfaebd5cfa5, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,513 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16838c5df0ab4e3938ad84e9a080def9, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,513 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2eeaafb9a1964f4c6899fa5dde05a463, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,513 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c98bb859cc9e047b211ddb4c5b967ce5, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,513 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b935ee7eeea02b161c0ebc0d05527e91, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,513 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4584d26a35c36968de829b34ac08fb5d, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,513 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b717f65da42dacb67935407f094e8467, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,513 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f2b6a6b11f14670aca9dccceb20d9630, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,513 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57995ad3e527ada56b846564833341d1, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,513 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c77db308a7c1c57fe9ef945e17e28127, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,513 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=782c52424de7d71cb43b8fbfe8e89f7f, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,513 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d34bdb8a1414fb25a475fb97fc440b4, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,513 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d86220c39cb75f305e7ed4d4c9f1191c, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,513 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be4cb5da32eefc543e09e80b426b6938, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,513 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4bf848f8297d89de7b203fcc6678eb5, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,514 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1202ee0cad3e428893fcdbc630d4f9b9, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,514 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=653572572648db72daea94dd65cfb7f8, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,514 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5fe5dc5eb9193cba5339ee508dff867, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,514 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=145c6ba0291376a96e3a37f6b274f63b, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,514 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5162d3c0b3255050cbeed27e4456e5c2, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,514 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d3ce5f64ef1889a3856db9d8c0f63a72, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,514 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60c948f12e02b6ab0066426b6aceea80, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=5 2024-12-12T06:12:39,514 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=764fc3b5431e53dfb445323d0bfd39a6, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,514 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=269d56dd3a6671f5c9a30401b73ade89, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,514 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae428798a6433b4d48a205ab3d14ba42, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,514 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=269eed2924060312949b51c10150f409, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,514 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-12T06:12:39,514 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733984019514 2024-12-12T06:12:39,514 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733984079514 2024-12-12T06:12:39,514 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-12-12T06:12:39,514 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-12T06:12:39,514 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-12T06:12:39,515 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-12T06:12:39,515 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-12T06:12:39,515 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-12T06:12:39,515 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-12T06:12:39,515 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-12T06:12:39,515 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-12T06:12:39,516 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-12T06:12:39,517 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-12T06:12:39,517 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-12T06:12:39,518 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-12T06:12:39,518 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=60c948f12e02b6ab0066426b6aceea80, REOPEN/MOVE 2024-12-12T06:12:39,518 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true 2024-12-12T06:12:39,518 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-12T06:12:39,518 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-12-12T06:12:39,519 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=65 2024-12-12T06:12:39,519 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ffbfd3107920,42275,1733983951224, table=hbase:meta, region=1588230740 2024-12-12T06:12:39,519 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ffbfd3107920,42275,1733983951224, state=OPEN 2024-12-12T06:12:39,531 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42275-0x10018b511340003, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:39,531 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46199-0x10018b511340001, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:39,531 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45209-0x10018b511340000, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-12T06:12:39,531 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:39,531 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:39,531 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-12T06:12:39,532 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733983989532 2024-12-12T06:12:39,532 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-12T06:12:39,532 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-12-12T06:12:39,533 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-12T06:12:39,534 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=60, state=ROLLEDBACK, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5d34bdb8a1414fb25a475fb97fc440b4, server=ffbfd3107920,42275,1733983951224:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ffbfd3107920,33455,1733983951151, splitWal=true, meta=true exec-time=2.4970 sec 2024-12-12T06:12:39,537 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8545c1fe97252abe3447dcfaebd5cfa5, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,537 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16838c5df0ab4e3938ad84e9a080def9, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,537 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2eeaafb9a1964f4c6899fa5dde05a463, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,537 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c98bb859cc9e047b211ddb4c5b967ce5, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,537 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b935ee7eeea02b161c0ebc0d05527e91, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,537 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4584d26a35c36968de829b34ac08fb5d, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,537 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b717f65da42dacb67935407f094e8467, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,537 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f2b6a6b11f14670aca9dccceb20d9630, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,537 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57995ad3e527ada56b846564833341d1, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,537 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c77db308a7c1c57fe9ef945e17e28127, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,537 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=782c52424de7d71cb43b8fbfe8e89f7f, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,538 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d34bdb8a1414fb25a475fb97fc440b4, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,538 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d86220c39cb75f305e7ed4d4c9f1191c, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,538 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be4cb5da32eefc543e09e80b426b6938, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,538 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4bf848f8297d89de7b203fcc6678eb5, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,538 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1202ee0cad3e428893fcdbc630d4f9b9, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,538 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=653572572648db72daea94dd65cfb7f8, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,538 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5fe5dc5eb9193cba5339ee508dff867, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,538 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=145c6ba0291376a96e3a37f6b274f63b, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,538 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5162d3c0b3255050cbeed27e4456e5c2, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,538 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d3ce5f64ef1889a3856db9d8c0f63a72, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,538 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60c948f12e02b6ab0066426b6aceea80, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=5 2024-12-12T06:12:39,538 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=764fc3b5431e53dfb445323d0bfd39a6, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,538 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=269d56dd3a6671f5c9a30401b73ade89, regionState=OPEN, lastHost=ffbfd3107920,46199,1733983951048, regionLocation=ffbfd3107920,46199,1733983951048, openSeqNum=2 2024-12-12T06:12:39,538 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae428798a6433b4d48a205ab3d14ba42, regionState=OPENING, lastHost=ffbfd3107920,33455,1733983951151, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,538 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=269eed2924060312949b51c10150f409, regionState=OPEN, lastHost=ffbfd3107920,42275,1733983951224, regionLocation=ffbfd3107920,42275,1733983951224, openSeqNum=2 2024-12-12T06:12:39,539 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-12T06:12:39,539 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733984019539 2024-12-12T06:12:39,539 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733984079539 2024-12-12T06:12:39,539 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-12-12T06:12:39,554 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.assignment.TestRollbackSCP#testFailAndRollback Thread=351 (was 385), OpenFileDescriptor=620 (was 643), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=65 (was 65), ProcessCount=11 (was 11), AvailableMemoryMB=9081 (was 9148) 2024-12-12T06:12:39,554 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-12T06:12:39,554 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-12T06:12:39,554 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-12T06:12:39,554 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-12T06:12:39,555 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-12T06:12:39,555 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-12T06:12:39,555 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-12T06:12:39,555 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=721888074, stopped=false 2024-12-12T06:12:39,555 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=ffbfd3107920,45209,1733983950336 2024-12-12T06:12:39,564 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45209-0x10018b511340000, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-12T06:12:39,564 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42275-0x10018b511340003, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-12T06:12:39,564 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46199-0x10018b511340001, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-12T06:12:39,564 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46199-0x10018b511340001, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-12T06:12:39,564 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42275-0x10018b511340003, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-12T06:12:39,564 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45209-0x10018b511340000, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-12T06:12:39,564 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-12T06:12:39,564 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-12T06:12:39,564 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-12T06:12:39,565 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-12T06:12:39,565 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ffbfd3107920,46199,1733983951048' ***** 2024-12-12T06:12:39,565 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-12T06:12:39,565 INFO [RS:0;ffbfd3107920:46199 {}] regionserver.HeapMemoryManager(224): Stopping 2024-12-12T06:12:39,565 INFO [RS:0;ffbfd3107920:46199 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-12T06:12:39,565 INFO [RS:0;ffbfd3107920:46199 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-12T06:12:39,565 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-12T06:12:39,565 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46199-0x10018b511340001, quorum=127.0.0.1:59802, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-12T06:12:39,565 INFO [RS:0;ffbfd3107920:46199 {}] regionserver.HRegionServer(3091): Received CLOSE for f5fe5dc5eb9193cba5339ee508dff867 2024-12-12T06:12:39,566 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42275-0x10018b511340003, quorum=127.0.0.1:59802, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-12T06:12:39,566 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45209-0x10018b511340000, quorum=127.0.0.1:59802, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-12T06:12:39,566 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ffbfd3107920,42275,1733983951224' ***** 2024-12-12T06:12:39,566 INFO [RS:0;ffbfd3107920:46199 {}] regionserver.HRegionServer(3091): Received CLOSE for d3ce5f64ef1889a3856db9d8c0f63a72 2024-12-12T06:12:39,566 INFO [RS:0;ffbfd3107920:46199 {}] regionserver.HRegionServer(3091): Received CLOSE for a4bf848f8297d89de7b203fcc6678eb5 2024-12-12T06:12:39,566 INFO [RS:0;ffbfd3107920:46199 {}] regionserver.HRegionServer(3091): Received CLOSE for 2eeaafb9a1964f4c6899fa5dde05a463 2024-12-12T06:12:39,566 INFO [RS:0;ffbfd3107920:46199 {}] regionserver.HRegionServer(3091): Received CLOSE for 57995ad3e527ada56b846564833341d1 2024-12-12T06:12:39,566 INFO [RS:0;ffbfd3107920:46199 {}] regionserver.HRegionServer(3091): Received CLOSE for c98bb859cc9e047b211ddb4c5b967ce5 2024-12-12T06:12:39,566 INFO [RS:0;ffbfd3107920:46199 {}] regionserver.HRegionServer(3091): Received CLOSE for 145c6ba0291376a96e3a37f6b274f63b 2024-12-12T06:12:39,566 INFO [RS:0;ffbfd3107920:46199 {}] regionserver.HRegionServer(3091): Received CLOSE for 269d56dd3a6671f5c9a30401b73ade89 2024-12-12T06:12:39,567 INFO [RS:0;ffbfd3107920:46199 {}] regionserver.HRegionServer(3091): Received CLOSE for b717f65da42dacb67935407f094e8467 2024-12-12T06:12:39,567 INFO [RS:0;ffbfd3107920:46199 {}] regionserver.HRegionServer(3091): Received CLOSE for 782c52424de7d71cb43b8fbfe8e89f7f 2024-12-12T06:12:39,567 INFO [RS:0;ffbfd3107920:46199 {}] regionserver.HRegionServer(3091): Received CLOSE for c77db308a7c1c57fe9ef945e17e28127 2024-12-12T06:12:39,567 INFO [RS:0;ffbfd3107920:46199 {}] regionserver.HRegionServer(959): stopping server ffbfd3107920,46199,1733983951048 2024-12-12T06:12:39,567 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f5fe5dc5eb9193cba5339ee508dff867, disabling compactions & flushes 2024-12-12T06:12:39,567 INFO [RS:0;ffbfd3107920:46199 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-12T06:12:39,567 INFO [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,rrr,1733983953560.f5fe5dc5eb9193cba5339ee508dff867. 2024-12-12T06:12:39,567 INFO [RS:0;ffbfd3107920:46199 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;ffbfd3107920:46199. 2024-12-12T06:12:39,567 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1733983953560.f5fe5dc5eb9193cba5339ee508dff867. 2024-12-12T06:12:39,567 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1733983953560.f5fe5dc5eb9193cba5339ee508dff867. after waiting 0 ms 2024-12-12T06:12:39,567 DEBUG [RS:0;ffbfd3107920:46199 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-12T06:12:39,567 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1733983953560.f5fe5dc5eb9193cba5339ee508dff867. 2024-12-12T06:12:39,567 DEBUG [RS:0;ffbfd3107920:46199 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-12T06:12:39,567 INFO [RS:0;ffbfd3107920:46199 {}] regionserver.HRegionServer(1321): Waiting on 11 regions to close 2024-12-12T06:12:39,567 DEBUG [RS:0;ffbfd3107920:46199 {}] regionserver.HRegionServer(1351): Waiting on 145c6ba0291376a96e3a37f6b274f63b, 269d56dd3a6671f5c9a30401b73ade89, 2eeaafb9a1964f4c6899fa5dde05a463, 57995ad3e527ada56b846564833341d1, 782c52424de7d71cb43b8fbfe8e89f7f, 8545c1fe97252abe3447dcfaebd5cfa5, a4bf848f8297d89de7b203fcc6678eb5, b717f65da42dacb67935407f094e8467, c77db308a7c1c57fe9ef945e17e28127, c98bb859cc9e047b211ddb4c5b967ce5, d3ce5f64ef1889a3856db9d8c0f63a72, f5fe5dc5eb9193cba5339ee508dff867 2024-12-12T06:12:39,567 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-12T06:12:39,568 INFO [RS:2;ffbfd3107920:42275 {}] regionserver.HeapMemoryManager(224): Stopping 2024-12-12T06:12:39,568 INFO [RS:2;ffbfd3107920:42275 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-12T06:12:39,568 INFO [RS:2;ffbfd3107920:42275 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-12T06:12:39,568 INFO [RS:2;ffbfd3107920:42275 {}] regionserver.HRegionServer(3091): Received CLOSE for 269eed2924060312949b51c10150f409 2024-12-12T06:12:39,568 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-12T06:12:39,568 INFO [RS:2;ffbfd3107920:42275 {}] regionserver.HRegionServer(3091): Received CLOSE for 5d34bdb8a1414fb25a475fb97fc440b4 2024-12-12T06:12:39,568 INFO [RS:2;ffbfd3107920:42275 {}] regionserver.HRegionServer(3091): Received CLOSE for 764fc3b5431e53dfb445323d0bfd39a6 2024-12-12T06:12:39,568 INFO [RS:2;ffbfd3107920:42275 {}] regionserver.HRegionServer(3091): Received CLOSE for 653572572648db72daea94dd65cfb7f8 2024-12-12T06:12:39,568 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 269eed2924060312949b51c10150f409, disabling compactions & flushes 2024-12-12T06:12:39,568 INFO [RS:2;ffbfd3107920:42275 {}] regionserver.HRegionServer(3091): Received CLOSE for 60c948f12e02b6ab0066426b6aceea80 2024-12-12T06:12:39,568 INFO [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,zzz,1733983953560.269eed2924060312949b51c10150f409. 2024-12-12T06:12:39,568 INFO [RS:2;ffbfd3107920:42275 {}] regionserver.HRegionServer(3091): Received CLOSE for be4cb5da32eefc543e09e80b426b6938 2024-12-12T06:12:39,568 INFO [RS:2;ffbfd3107920:42275 {}] regionserver.HRegionServer(3091): Received CLOSE for 16838c5df0ab4e3938ad84e9a080def9 2024-12-12T06:12:39,568 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1733983953560.269eed2924060312949b51c10150f409. 2024-12-12T06:12:39,568 INFO [RS:2;ffbfd3107920:42275 {}] regionserver.HRegionServer(3091): Received CLOSE for 4584d26a35c36968de829b34ac08fb5d 2024-12-12T06:12:39,568 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1733983953560.269eed2924060312949b51c10150f409. after waiting 0 ms 2024-12-12T06:12:39,568 INFO [RS:2;ffbfd3107920:42275 {}] regionserver.HRegionServer(3091): Received CLOSE for b935ee7eeea02b161c0ebc0d05527e91 2024-12-12T06:12:39,568 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1733983953560.269eed2924060312949b51c10150f409. 2024-12-12T06:12:39,568 INFO [RS:2;ffbfd3107920:42275 {}] regionserver.HRegionServer(3091): Received CLOSE for 5162d3c0b3255050cbeed27e4456e5c2 2024-12-12T06:12:39,569 INFO [RS:2;ffbfd3107920:42275 {}] regionserver.HRegionServer(3091): Received CLOSE for f2b6a6b11f14670aca9dccceb20d9630 2024-12-12T06:12:39,569 INFO [RS:2;ffbfd3107920:42275 {}] regionserver.HRegionServer(959): stopping server ffbfd3107920,42275,1733983951224 2024-12-12T06:12:39,569 INFO [RS:2;ffbfd3107920:42275 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-12T06:12:39,569 INFO [RS:2;ffbfd3107920:42275 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;ffbfd3107920:42275. 2024-12-12T06:12:39,569 DEBUG [RS:2;ffbfd3107920:42275 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-12T06:12:39,569 DEBUG [RS:2;ffbfd3107920:42275 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-12T06:12:39,569 INFO [RS:2;ffbfd3107920:42275 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-12T06:12:39,569 INFO [RS:2;ffbfd3107920:42275 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-12T06:12:39,569 INFO [RS:2;ffbfd3107920:42275 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-12T06:12:39,569 INFO [RS:2;ffbfd3107920:42275 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-12T06:12:39,570 INFO [RS:2;ffbfd3107920:42275 {}] regionserver.HRegionServer(1321): Waiting on 12 regions to close 2024-12-12T06:12:39,570 DEBUG [RS:2;ffbfd3107920:42275 {}] regionserver.HRegionServer(1351): Waiting on 1202ee0cad3e428893fcdbc630d4f9b9, 1588230740, 16838c5df0ab4e3938ad84e9a080def9, 269eed2924060312949b51c10150f409, 4584d26a35c36968de829b34ac08fb5d, 5162d3c0b3255050cbeed27e4456e5c2, 5d34bdb8a1414fb25a475fb97fc440b4, 60c948f12e02b6ab0066426b6aceea80, 653572572648db72daea94dd65cfb7f8, 764fc3b5431e53dfb445323d0bfd39a6, b935ee7eeea02b161c0ebc0d05527e91, be4cb5da32eefc543e09e80b426b6938, f2b6a6b11f14670aca9dccceb20d9630 2024-12-12T06:12:39,570 INFO [regionserver/ffbfd3107920:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-12T06:12:39,570 INFO [regionserver/ffbfd3107920:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-12T06:12:39,570 INFO [regionserver/ffbfd3107920:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-12T06:12:39,570 INFO [regionserver/ffbfd3107920:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-12T06:12:39,570 DEBUG [RS_CLOSE_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-12T06:12:39,570 INFO [RS_CLOSE_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-12T06:12:39,570 DEBUG [RS_CLOSE_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-12T06:12:39,570 DEBUG [RS_CLOSE_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-12T06:12:39,570 DEBUG [RS_CLOSE_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-12T06:12:39,570 INFO [RS_CLOSE_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.64 KB heapSize=5.42 KB 2024-12-12T06:12:39,572 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/f5fe5dc5eb9193cba5339ee508dff867/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-12T06:12:39,573 INFO [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,rrr,1733983953560.f5fe5dc5eb9193cba5339ee508dff867. 2024-12-12T06:12:39,573 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f5fe5dc5eb9193cba5339ee508dff867: Waiting for close lock at 1733983959566Running coprocessor pre-close hooks at 1733983959567 (+1 ms)Disabling compacts and flushes for region at 1733983959567Disabling writes for close at 1733983959567Writing region close event to WAL at 1733983959567Running coprocessor post-close hooks at 1733983959573 (+6 ms)Closed at 1733983959573 2024-12-12T06:12:39,574 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,rrr,1733983953560.f5fe5dc5eb9193cba5339ee508dff867. 2024-12-12T06:12:39,574 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d3ce5f64ef1889a3856db9d8c0f63a72, disabling compactions & flushes 2024-12-12T06:12:39,574 INFO [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,uuu,1733983953560.d3ce5f64ef1889a3856db9d8c0f63a72. 2024-12-12T06:12:39,574 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1733983953560.d3ce5f64ef1889a3856db9d8c0f63a72. 2024-12-12T06:12:39,574 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1733983953560.d3ce5f64ef1889a3856db9d8c0f63a72. after waiting 0 ms 2024-12-12T06:12:39,574 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1733983953560.d3ce5f64ef1889a3856db9d8c0f63a72. 2024-12-12T06:12:39,576 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/269eed2924060312949b51c10150f409/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-12T06:12:39,577 INFO [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,zzz,1733983953560.269eed2924060312949b51c10150f409. 2024-12-12T06:12:39,577 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 269eed2924060312949b51c10150f409: Waiting for close lock at 1733983959568Running coprocessor pre-close hooks at 1733983959568Disabling compacts and flushes for region at 1733983959568Disabling writes for close at 1733983959568Writing region close event to WAL at 1733983959571 (+3 ms)Running coprocessor post-close hooks at 1733983959577 (+6 ms)Closed at 1733983959577 2024-12-12T06:12:39,578 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,zzz,1733983953560.269eed2924060312949b51c10150f409. 2024-12-12T06:12:39,578 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5d34bdb8a1414fb25a475fb97fc440b4, disabling compactions & flushes 2024-12-12T06:12:39,578 INFO [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,lll,1733983953560.5d34bdb8a1414fb25a475fb97fc440b4. 2024-12-12T06:12:39,578 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1733983953560.5d34bdb8a1414fb25a475fb97fc440b4. 2024-12-12T06:12:39,578 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,lll,1733983953560.5d34bdb8a1414fb25a475fb97fc440b4. after waiting 0 ms 2024-12-12T06:12:39,578 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,lll,1733983953560.5d34bdb8a1414fb25a475fb97fc440b4. 2024-12-12T06:12:39,580 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/d3ce5f64ef1889a3856db9d8c0f63a72/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-12T06:12:39,580 INFO [regionserver/ffbfd3107920:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-12T06:12:39,581 INFO [regionserver/ffbfd3107920:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-12T06:12:39,581 INFO [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,uuu,1733983953560.d3ce5f64ef1889a3856db9d8c0f63a72. 2024-12-12T06:12:39,581 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d3ce5f64ef1889a3856db9d8c0f63a72: Waiting for close lock at 1733983959574Running coprocessor pre-close hooks at 1733983959574Disabling compacts and flushes for region at 1733983959574Disabling writes for close at 1733983959574Writing region close event to WAL at 1733983959574Running coprocessor post-close hooks at 1733983959581 (+7 ms)Closed at 1733983959581 2024-12-12T06:12:39,581 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,uuu,1733983953560.d3ce5f64ef1889a3856db9d8c0f63a72. 2024-12-12T06:12:39,581 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a4bf848f8297d89de7b203fcc6678eb5, disabling compactions & flushes 2024-12-12T06:12:39,581 INFO [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ooo,1733983953560.a4bf848f8297d89de7b203fcc6678eb5. 2024-12-12T06:12:39,581 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1733983953560.a4bf848f8297d89de7b203fcc6678eb5. 2024-12-12T06:12:39,582 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1733983953560.a4bf848f8297d89de7b203fcc6678eb5. after waiting 0 ms 2024-12-12T06:12:39,582 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1733983953560.a4bf848f8297d89de7b203fcc6678eb5. 2024-12-12T06:12:39,583 DEBUG [RS_CLOSE_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/hbase/meta/1588230740/.tmp/info/0dcbe5e7c75746bdabe86df14e4e8470 is 138, key is test,fff,1733983953560.4584d26a35c36968de829b34ac08fb5d./info:regioninfo/1733983958416/Put/seqid=0 2024-12-12T06:12:39,587 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/5d34bdb8a1414fb25a475fb97fc440b4/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-12T06:12:39,588 INFO [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,lll,1733983953560.5d34bdb8a1414fb25a475fb97fc440b4. 2024-12-12T06:12:39,588 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5d34bdb8a1414fb25a475fb97fc440b4: Waiting for close lock at 1733983959578Running coprocessor pre-close hooks at 1733983959578Disabling compacts and flushes for region at 1733983959578Disabling writes for close at 1733983959578Writing region close event to WAL at 1733983959579 (+1 ms)Running coprocessor post-close hooks at 1733983959588 (+9 ms)Closed at 1733983959588 2024-12-12T06:12:39,588 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,lll,1733983953560.5d34bdb8a1414fb25a475fb97fc440b4. 2024-12-12T06:12:39,589 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 764fc3b5431e53dfb445323d0bfd39a6, disabling compactions & flushes 2024-12-12T06:12:39,589 INFO [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,www,1733983953560.764fc3b5431e53dfb445323d0bfd39a6. 2024-12-12T06:12:39,589 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1733983953560.764fc3b5431e53dfb445323d0bfd39a6. 2024-12-12T06:12:39,589 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,www,1733983953560.764fc3b5431e53dfb445323d0bfd39a6. after waiting 0 ms 2024-12-12T06:12:39,589 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,www,1733983953560.764fc3b5431e53dfb445323d0bfd39a6. 2024-12-12T06:12:39,592 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/a4bf848f8297d89de7b203fcc6678eb5/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-12T06:12:39,592 INFO [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ooo,1733983953560.a4bf848f8297d89de7b203fcc6678eb5. 2024-12-12T06:12:39,592 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a4bf848f8297d89de7b203fcc6678eb5: Waiting for close lock at 1733983959581Running coprocessor pre-close hooks at 1733983959581Disabling compacts and flushes for region at 1733983959581Disabling writes for close at 1733983959582 (+1 ms)Writing region close event to WAL at 1733983959583 (+1 ms)Running coprocessor post-close hooks at 1733983959592 (+9 ms)Closed at 1733983959592 2024-12-12T06:12:39,592 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ooo,1733983953560.a4bf848f8297d89de7b203fcc6678eb5. 2024-12-12T06:12:39,593 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2eeaafb9a1964f4c6899fa5dde05a463, disabling compactions & flushes 2024-12-12T06:12:39,593 INFO [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ccc,1733983953560.2eeaafb9a1964f4c6899fa5dde05a463. 2024-12-12T06:12:39,593 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1733983953560.2eeaafb9a1964f4c6899fa5dde05a463. 2024-12-12T06:12:39,593 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1733983953560.2eeaafb9a1964f4c6899fa5dde05a463. after waiting 0 ms 2024-12-12T06:12:39,593 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1733983953560.2eeaafb9a1964f4c6899fa5dde05a463. 2024-12-12T06:12:39,600 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/764fc3b5431e53dfb445323d0bfd39a6/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-12T06:12:39,601 INFO [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,www,1733983953560.764fc3b5431e53dfb445323d0bfd39a6. 2024-12-12T06:12:39,601 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 764fc3b5431e53dfb445323d0bfd39a6: Waiting for close lock at 1733983959588Running coprocessor pre-close hooks at 1733983959588Disabling compacts and flushes for region at 1733983959588Disabling writes for close at 1733983959589 (+1 ms)Writing region close event to WAL at 1733983959593 (+4 ms)Running coprocessor post-close hooks at 1733983959601 (+8 ms)Closed at 1733983959601 2024-12-12T06:12:39,601 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,www,1733983953560.764fc3b5431e53dfb445323d0bfd39a6. 2024-12-12T06:12:39,601 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 653572572648db72daea94dd65cfb7f8, disabling compactions & flushes 2024-12-12T06:12:39,601 INFO [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,qqq,1733983953560.653572572648db72daea94dd65cfb7f8. 2024-12-12T06:12:39,601 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1733983953560.653572572648db72daea94dd65cfb7f8. 2024-12-12T06:12:39,601 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1733983953560.653572572648db72daea94dd65cfb7f8. after waiting 0 ms 2024-12-12T06:12:39,601 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1733983953560.653572572648db72daea94dd65cfb7f8. 2024-12-12T06:12:39,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43493 is added to blk_1073741869_1045 (size=8177) 2024-12-12T06:12:39,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38541 is added to blk_1073741869_1045 (size=8177) 2024-12-12T06:12:39,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42285 is added to blk_1073741869_1045 (size=8177) 2024-12-12T06:12:39,608 INFO [RS_CLOSE_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.64 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/hbase/meta/1588230740/.tmp/info/0dcbe5e7c75746bdabe86df14e4e8470 2024-12-12T06:12:39,611 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/2eeaafb9a1964f4c6899fa5dde05a463/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-12T06:12:39,612 INFO [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ccc,1733983953560.2eeaafb9a1964f4c6899fa5dde05a463. 2024-12-12T06:12:39,612 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2eeaafb9a1964f4c6899fa5dde05a463: Waiting for close lock at 1733983959592Running coprocessor pre-close hooks at 1733983959592Disabling compacts and flushes for region at 1733983959592Disabling writes for close at 1733983959593 (+1 ms)Writing region close event to WAL at 1733983959599 (+6 ms)Running coprocessor post-close hooks at 1733983959612 (+13 ms)Closed at 1733983959612 2024-12-12T06:12:39,612 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ccc,1733983953560.2eeaafb9a1964f4c6899fa5dde05a463. 2024-12-12T06:12:39,613 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 57995ad3e527ada56b846564833341d1, disabling compactions & flushes 2024-12-12T06:12:39,613 INFO [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,iii,1733983953560.57995ad3e527ada56b846564833341d1. 2024-12-12T06:12:39,613 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1733983953560.57995ad3e527ada56b846564833341d1. 2024-12-12T06:12:39,613 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,iii,1733983953560.57995ad3e527ada56b846564833341d1. after waiting 0 ms 2024-12-12T06:12:39,613 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,iii,1733983953560.57995ad3e527ada56b846564833341d1. 2024-12-12T06:12:39,616 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/653572572648db72daea94dd65cfb7f8/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-12T06:12:39,616 DEBUG [RS_CLOSE_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/hbase/meta/1588230740/.tmp/info/0dcbe5e7c75746bdabe86df14e4e8470 as hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/hbase/meta/1588230740/info/0dcbe5e7c75746bdabe86df14e4e8470 2024-12-12T06:12:39,617 INFO [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,qqq,1733983953560.653572572648db72daea94dd65cfb7f8. 2024-12-12T06:12:39,617 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 653572572648db72daea94dd65cfb7f8: Waiting for close lock at 1733983959601Running coprocessor pre-close hooks at 1733983959601Disabling compacts and flushes for region at 1733983959601Disabling writes for close at 1733983959601Writing region close event to WAL at 1733983959604 (+3 ms)Running coprocessor post-close hooks at 1733983959617 (+13 ms)Closed at 1733983959617 2024-12-12T06:12:39,617 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,qqq,1733983953560.653572572648db72daea94dd65cfb7f8. 2024-12-12T06:12:39,617 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 60c948f12e02b6ab0066426b6aceea80, disabling compactions & flushes 2024-12-12T06:12:39,617 INFO [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,vvv,1733983953560.60c948f12e02b6ab0066426b6aceea80. 2024-12-12T06:12:39,617 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1733983953560.60c948f12e02b6ab0066426b6aceea80. 2024-12-12T06:12:39,617 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1733983953560.60c948f12e02b6ab0066426b6aceea80. after waiting 0 ms 2024-12-12T06:12:39,617 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1733983953560.60c948f12e02b6ab0066426b6aceea80. 2024-12-12T06:12:39,620 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/57995ad3e527ada56b846564833341d1/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-12T06:12:39,621 INFO [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,iii,1733983953560.57995ad3e527ada56b846564833341d1. 2024-12-12T06:12:39,621 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 57995ad3e527ada56b846564833341d1: Waiting for close lock at 1733983959613Running coprocessor pre-close hooks at 1733983959613Disabling compacts and flushes for region at 1733983959613Disabling writes for close at 1733983959613Writing region close event to WAL at 1733983959615 (+2 ms)Running coprocessor post-close hooks at 1733983959620 (+5 ms)Closed at 1733983959621 (+1 ms) 2024-12-12T06:12:39,621 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,iii,1733983953560.57995ad3e527ada56b846564833341d1. 2024-12-12T06:12:39,621 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c98bb859cc9e047b211ddb4c5b967ce5, disabling compactions & flushes 2024-12-12T06:12:39,621 INFO [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ddd,1733983953560.c98bb859cc9e047b211ddb4c5b967ce5. 2024-12-12T06:12:39,621 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1733983953560.c98bb859cc9e047b211ddb4c5b967ce5. 2024-12-12T06:12:39,621 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1733983953560.c98bb859cc9e047b211ddb4c5b967ce5. after waiting 0 ms 2024-12-12T06:12:39,621 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1733983953560.c98bb859cc9e047b211ddb4c5b967ce5. 2024-12-12T06:12:39,623 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/60c948f12e02b6ab0066426b6aceea80/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-12-12T06:12:39,625 INFO [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,vvv,1733983953560.60c948f12e02b6ab0066426b6aceea80. 2024-12-12T06:12:39,625 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 60c948f12e02b6ab0066426b6aceea80: Waiting for close lock at 1733983959617Running coprocessor pre-close hooks at 1733983959617Disabling compacts and flushes for region at 1733983959617Disabling writes for close at 1733983959617Writing region close event to WAL at 1733983959618 (+1 ms)Running coprocessor post-close hooks at 1733983959625 (+7 ms)Closed at 1733983959625 2024-12-12T06:12:39,625 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,vvv,1733983953560.60c948f12e02b6ab0066426b6aceea80. 2024-12-12T06:12:39,625 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing be4cb5da32eefc543e09e80b426b6938, disabling compactions & flushes 2024-12-12T06:12:39,625 INFO [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,nnn,1733983953560.be4cb5da32eefc543e09e80b426b6938. 2024-12-12T06:12:39,625 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1733983953560.be4cb5da32eefc543e09e80b426b6938. 2024-12-12T06:12:39,625 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1733983953560.be4cb5da32eefc543e09e80b426b6938. after waiting 0 ms 2024-12-12T06:12:39,626 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1733983953560.be4cb5da32eefc543e09e80b426b6938. 2024-12-12T06:12:39,626 INFO [RS_CLOSE_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/hbase/meta/1588230740/info/0dcbe5e7c75746bdabe86df14e4e8470, entries=24, sequenceid=75, filesize=8.0 K 2024-12-12T06:12:39,627 INFO [RS_CLOSE_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.64 KB/2708, heapSize ~4.66 KB/4768, currentSize=0 B/0 for 1588230740 in 57ms, sequenceid=75, compaction requested=false 2024-12-12T06:12:39,628 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/c98bb859cc9e047b211ddb4c5b967ce5/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-12T06:12:39,628 INFO [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ddd,1733983953560.c98bb859cc9e047b211ddb4c5b967ce5. 2024-12-12T06:12:39,629 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c98bb859cc9e047b211ddb4c5b967ce5: Waiting for close lock at 1733983959621Running coprocessor pre-close hooks at 1733983959621Disabling compacts and flushes for region at 1733983959621Disabling writes for close at 1733983959621Writing region close event to WAL at 1733983959622 (+1 ms)Running coprocessor post-close hooks at 1733983959628 (+6 ms)Closed at 1733983959628 2024-12-12T06:12:39,629 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ddd,1733983953560.c98bb859cc9e047b211ddb4c5b967ce5. 2024-12-12T06:12:39,629 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 145c6ba0291376a96e3a37f6b274f63b, disabling compactions & flushes 2024-12-12T06:12:39,629 INFO [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,sss,1733983953560.145c6ba0291376a96e3a37f6b274f63b. 2024-12-12T06:12:39,629 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1733983953560.145c6ba0291376a96e3a37f6b274f63b. 2024-12-12T06:12:39,629 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,sss,1733983953560.145c6ba0291376a96e3a37f6b274f63b. after waiting 0 ms 2024-12-12T06:12:39,629 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,sss,1733983953560.145c6ba0291376a96e3a37f6b274f63b. 2024-12-12T06:12:39,631 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/be4cb5da32eefc543e09e80b426b6938/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-12T06:12:39,632 INFO [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,nnn,1733983953560.be4cb5da32eefc543e09e80b426b6938. 2024-12-12T06:12:39,632 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for be4cb5da32eefc543e09e80b426b6938: Waiting for close lock at 1733983959625Running coprocessor pre-close hooks at 1733983959625Disabling compacts and flushes for region at 1733983959625Disabling writes for close at 1733983959625Writing region close event to WAL at 1733983959626 (+1 ms)Running coprocessor post-close hooks at 1733983959632 (+6 ms)Closed at 1733983959632 2024-12-12T06:12:39,633 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,nnn,1733983953560.be4cb5da32eefc543e09e80b426b6938. 2024-12-12T06:12:39,633 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 16838c5df0ab4e3938ad84e9a080def9, disabling compactions & flushes 2024-12-12T06:12:39,633 INFO [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,bbb,1733983953560.16838c5df0ab4e3938ad84e9a080def9. 2024-12-12T06:12:39,633 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1733983953560.16838c5df0ab4e3938ad84e9a080def9. 2024-12-12T06:12:39,633 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1733983953560.16838c5df0ab4e3938ad84e9a080def9. after waiting 0 ms 2024-12-12T06:12:39,633 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1733983953560.16838c5df0ab4e3938ad84e9a080def9. 2024-12-12T06:12:39,635 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/145c6ba0291376a96e3a37f6b274f63b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-12T06:12:39,635 DEBUG [RS_CLOSE_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/hbase/meta/1588230740/recovered.edits/78.seqid, newMaxSeqId=78, maxSeqId=64 2024-12-12T06:12:39,636 DEBUG [RS_CLOSE_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-12T06:12:39,636 INFO [RS_CLOSE_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-12T06:12:39,636 INFO [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,sss,1733983953560.145c6ba0291376a96e3a37f6b274f63b. 2024-12-12T06:12:39,636 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 145c6ba0291376a96e3a37f6b274f63b: Waiting for close lock at 1733983959629Running coprocessor pre-close hooks at 1733983959629Disabling compacts and flushes for region at 1733983959629Disabling writes for close at 1733983959629Writing region close event to WAL at 1733983959630 (+1 ms)Running coprocessor post-close hooks at 1733983959636 (+6 ms)Closed at 1733983959636 2024-12-12T06:12:39,636 DEBUG [RS_CLOSE_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733983959570Running coprocessor pre-close hooks at 1733983959570Disabling compacts and flushes for region at 1733983959570Disabling writes for close at 1733983959570Obtaining lock to block concurrent updates at 1733983959570Preparing flush snapshotting stores in 1588230740 at 1733983959570Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=2708, getHeapSize=5488, getOffHeapSize=0, getCellsCount=24 at 1733983959577 (+7 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733983959578 (+1 ms)Flushing 1588230740/info: creating writer at 1733983959578Flushing 1588230740/info: appending metadata at 1733983959582 (+4 ms)Flushing 1588230740/info: closing flushed file at 1733983959583 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@797033ac: reopening flushed file at 1733983959615 (+32 ms)Finished flush of dataSize ~2.64 KB/2708, heapSize ~4.66 KB/4768, currentSize=0 B/0 for 1588230740 in 57ms, sequenceid=75, compaction requested=false at 1733983959627 (+12 ms)Writing region close event to WAL at 1733983959631 (+4 ms)Running coprocessor post-close hooks at 1733983959636 (+5 ms)Closed at 1733983959636 2024-12-12T06:12:39,636 DEBUG [RS_CLOSE_META-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-12T06:12:39,636 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,sss,1733983953560.145c6ba0291376a96e3a37f6b274f63b. 2024-12-12T06:12:39,637 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 269d56dd3a6671f5c9a30401b73ade89, disabling compactions & flushes 2024-12-12T06:12:39,637 INFO [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,xxx,1733983953560.269d56dd3a6671f5c9a30401b73ade89. 2024-12-12T06:12:39,637 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1733983953560.269d56dd3a6671f5c9a30401b73ade89. 2024-12-12T06:12:39,637 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1733983953560.269d56dd3a6671f5c9a30401b73ade89. after waiting 0 ms 2024-12-12T06:12:39,637 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1733983953560.269d56dd3a6671f5c9a30401b73ade89. 2024-12-12T06:12:39,638 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/16838c5df0ab4e3938ad84e9a080def9/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-12T06:12:39,639 INFO [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,bbb,1733983953560.16838c5df0ab4e3938ad84e9a080def9. 2024-12-12T06:12:39,639 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 16838c5df0ab4e3938ad84e9a080def9: Waiting for close lock at 1733983959633Running coprocessor pre-close hooks at 1733983959633Disabling compacts and flushes for region at 1733983959633Disabling writes for close at 1733983959633Writing region close event to WAL at 1733983959634 (+1 ms)Running coprocessor post-close hooks at 1733983959639 (+5 ms)Closed at 1733983959639 2024-12-12T06:12:39,639 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,bbb,1733983953560.16838c5df0ab4e3938ad84e9a080def9. 2024-12-12T06:12:39,639 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4584d26a35c36968de829b34ac08fb5d, disabling compactions & flushes 2024-12-12T06:12:39,639 INFO [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,fff,1733983953560.4584d26a35c36968de829b34ac08fb5d. 2024-12-12T06:12:39,639 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1733983953560.4584d26a35c36968de829b34ac08fb5d. 2024-12-12T06:12:39,640 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,fff,1733983953560.4584d26a35c36968de829b34ac08fb5d. after waiting 0 ms 2024-12-12T06:12:39,640 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,fff,1733983953560.4584d26a35c36968de829b34ac08fb5d. 2024-12-12T06:12:39,641 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/269d56dd3a6671f5c9a30401b73ade89/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-12T06:12:39,642 INFO [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,xxx,1733983953560.269d56dd3a6671f5c9a30401b73ade89. 2024-12-12T06:12:39,642 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 269d56dd3a6671f5c9a30401b73ade89: Waiting for close lock at 1733983959637Running coprocessor pre-close hooks at 1733983959637Disabling compacts and flushes for region at 1733983959637Disabling writes for close at 1733983959637Writing region close event to WAL at 1733983959637Running coprocessor post-close hooks at 1733983959642 (+5 ms)Closed at 1733983959642 2024-12-12T06:12:39,642 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,xxx,1733983953560.269d56dd3a6671f5c9a30401b73ade89. 2024-12-12T06:12:39,642 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b717f65da42dacb67935407f094e8467, disabling compactions & flushes 2024-12-12T06:12:39,642 INFO [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ggg,1733983953560.b717f65da42dacb67935407f094e8467. 2024-12-12T06:12:39,642 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1733983953560.b717f65da42dacb67935407f094e8467. 2024-12-12T06:12:39,642 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1733983953560.b717f65da42dacb67935407f094e8467. after waiting 0 ms 2024-12-12T06:12:39,642 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1733983953560.b717f65da42dacb67935407f094e8467. 2024-12-12T06:12:39,644 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/4584d26a35c36968de829b34ac08fb5d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-12T06:12:39,644 INFO [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,fff,1733983953560.4584d26a35c36968de829b34ac08fb5d. 2024-12-12T06:12:39,644 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4584d26a35c36968de829b34ac08fb5d: Waiting for close lock at 1733983959639Running coprocessor pre-close hooks at 1733983959639Disabling compacts and flushes for region at 1733983959639Disabling writes for close at 1733983959640 (+1 ms)Writing region close event to WAL at 1733983959640Running coprocessor post-close hooks at 1733983959644 (+4 ms)Closed at 1733983959644 2024-12-12T06:12:39,645 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,fff,1733983953560.4584d26a35c36968de829b34ac08fb5d. 2024-12-12T06:12:39,645 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b935ee7eeea02b161c0ebc0d05527e91, disabling compactions & flushes 2024-12-12T06:12:39,645 INFO [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,eee,1733983953560.b935ee7eeea02b161c0ebc0d05527e91. 2024-12-12T06:12:39,645 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1733983953560.b935ee7eeea02b161c0ebc0d05527e91. 2024-12-12T06:12:39,645 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,eee,1733983953560.b935ee7eeea02b161c0ebc0d05527e91. after waiting 0 ms 2024-12-12T06:12:39,645 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,eee,1733983953560.b935ee7eeea02b161c0ebc0d05527e91. 2024-12-12T06:12:39,647 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/b717f65da42dacb67935407f094e8467/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-12T06:12:39,647 INFO [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ggg,1733983953560.b717f65da42dacb67935407f094e8467. 2024-12-12T06:12:39,647 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b717f65da42dacb67935407f094e8467: Waiting for close lock at 1733983959642Running coprocessor pre-close hooks at 1733983959642Disabling compacts and flushes for region at 1733983959642Disabling writes for close at 1733983959642Writing region close event to WAL at 1733983959643 (+1 ms)Running coprocessor post-close hooks at 1733983959647 (+4 ms)Closed at 1733983959647 2024-12-12T06:12:39,648 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ggg,1733983953560.b717f65da42dacb67935407f094e8467. 2024-12-12T06:12:39,648 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 782c52424de7d71cb43b8fbfe8e89f7f, disabling compactions & flushes 2024-12-12T06:12:39,648 INFO [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,kkk,1733983953560.782c52424de7d71cb43b8fbfe8e89f7f. 2024-12-12T06:12:39,648 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1733983953560.782c52424de7d71cb43b8fbfe8e89f7f. 2024-12-12T06:12:39,648 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1733983953560.782c52424de7d71cb43b8fbfe8e89f7f. after waiting 0 ms 2024-12-12T06:12:39,648 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1733983953560.782c52424de7d71cb43b8fbfe8e89f7f. 2024-12-12T06:12:39,650 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/b935ee7eeea02b161c0ebc0d05527e91/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-12T06:12:39,650 INFO [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,eee,1733983953560.b935ee7eeea02b161c0ebc0d05527e91. 2024-12-12T06:12:39,651 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b935ee7eeea02b161c0ebc0d05527e91: Waiting for close lock at 1733983959645Running coprocessor pre-close hooks at 1733983959645Disabling compacts and flushes for region at 1733983959645Disabling writes for close at 1733983959645Writing region close event to WAL at 1733983959645Running coprocessor post-close hooks at 1733983959650 (+5 ms)Closed at 1733983959650 2024-12-12T06:12:39,651 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,eee,1733983953560.b935ee7eeea02b161c0ebc0d05527e91. 2024-12-12T06:12:39,651 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5162d3c0b3255050cbeed27e4456e5c2, disabling compactions & flushes 2024-12-12T06:12:39,651 INFO [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ttt,1733983953560.5162d3c0b3255050cbeed27e4456e5c2. 2024-12-12T06:12:39,651 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1733983953560.5162d3c0b3255050cbeed27e4456e5c2. 2024-12-12T06:12:39,651 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1733983953560.5162d3c0b3255050cbeed27e4456e5c2. after waiting 0 ms 2024-12-12T06:12:39,651 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1733983953560.5162d3c0b3255050cbeed27e4456e5c2. 2024-12-12T06:12:39,652 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/782c52424de7d71cb43b8fbfe8e89f7f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-12T06:12:39,653 INFO [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,kkk,1733983953560.782c52424de7d71cb43b8fbfe8e89f7f. 2024-12-12T06:12:39,653 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 782c52424de7d71cb43b8fbfe8e89f7f: Waiting for close lock at 1733983959648Running coprocessor pre-close hooks at 1733983959648Disabling compacts and flushes for region at 1733983959648Disabling writes for close at 1733983959648Writing region close event to WAL at 1733983959648Running coprocessor post-close hooks at 1733983959653 (+5 ms)Closed at 1733983959653 2024-12-12T06:12:39,653 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,kkk,1733983953560.782c52424de7d71cb43b8fbfe8e89f7f. 2024-12-12T06:12:39,654 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c77db308a7c1c57fe9ef945e17e28127, disabling compactions & flushes 2024-12-12T06:12:39,654 INFO [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,jjj,1733983953560.c77db308a7c1c57fe9ef945e17e28127. 2024-12-12T06:12:39,654 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1733983953560.c77db308a7c1c57fe9ef945e17e28127. 2024-12-12T06:12:39,654 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1733983953560.c77db308a7c1c57fe9ef945e17e28127. after waiting 0 ms 2024-12-12T06:12:39,654 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1733983953560.c77db308a7c1c57fe9ef945e17e28127. 2024-12-12T06:12:39,655 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/5162d3c0b3255050cbeed27e4456e5c2/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-12T06:12:39,656 INFO [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ttt,1733983953560.5162d3c0b3255050cbeed27e4456e5c2. 2024-12-12T06:12:39,656 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5162d3c0b3255050cbeed27e4456e5c2: Waiting for close lock at 1733983959651Running coprocessor pre-close hooks at 1733983959651Disabling compacts and flushes for region at 1733983959651Disabling writes for close at 1733983959651Writing region close event to WAL at 1733983959652 (+1 ms)Running coprocessor post-close hooks at 1733983959656 (+4 ms)Closed at 1733983959656 2024-12-12T06:12:39,656 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ttt,1733983953560.5162d3c0b3255050cbeed27e4456e5c2. 2024-12-12T06:12:39,656 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f2b6a6b11f14670aca9dccceb20d9630, disabling compactions & flushes 2024-12-12T06:12:39,656 INFO [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,hhh,1733983953560.f2b6a6b11f14670aca9dccceb20d9630. 2024-12-12T06:12:39,656 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1733983953560.f2b6a6b11f14670aca9dccceb20d9630. 2024-12-12T06:12:39,657 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1733983953560.f2b6a6b11f14670aca9dccceb20d9630. after waiting 0 ms 2024-12-12T06:12:39,657 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1733983953560.f2b6a6b11f14670aca9dccceb20d9630. 2024-12-12T06:12:39,658 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/c77db308a7c1c57fe9ef945e17e28127/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-12T06:12:39,658 INFO [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,jjj,1733983953560.c77db308a7c1c57fe9ef945e17e28127. 2024-12-12T06:12:39,659 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c77db308a7c1c57fe9ef945e17e28127: Waiting for close lock at 1733983959653Running coprocessor pre-close hooks at 1733983959654 (+1 ms)Disabling compacts and flushes for region at 1733983959654Disabling writes for close at 1733983959654Writing region close event to WAL at 1733983959654Running coprocessor post-close hooks at 1733983959658 (+4 ms)Closed at 1733983959658 2024-12-12T06:12:39,659 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,jjj,1733983953560.c77db308a7c1c57fe9ef945e17e28127. 2024-12-12T06:12:39,660 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/f2b6a6b11f14670aca9dccceb20d9630/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-12T06:12:39,667 INFO [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,hhh,1733983953560.f2b6a6b11f14670aca9dccceb20d9630. 2024-12-12T06:12:39,667 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f2b6a6b11f14670aca9dccceb20d9630: Waiting for close lock at 1733983959656Running coprocessor pre-close hooks at 1733983959656Disabling compacts and flushes for region at 1733983959656Disabling writes for close at 1733983959657 (+1 ms)Writing region close event to WAL at 1733983959657Running coprocessor post-close hooks at 1733983959666 (+9 ms)Closed at 1733983959666 2024-12-12T06:12:39,667 DEBUG [RS_CLOSE_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,hhh,1733983953560.f2b6a6b11f14670aca9dccceb20d9630. 2024-12-12T06:12:39,768 INFO [RS:0;ffbfd3107920:46199 {}] regionserver.HRegionServer(976): stopping server ffbfd3107920,46199,1733983951048; all regions closed. 2024-12-12T06:12:39,770 INFO [RS:2;ffbfd3107920:42275 {}] regionserver.HRegionServer(976): stopping server ffbfd3107920,42275,1733983951224; all regions closed. 2024-12-12T06:12:39,778 WARN [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,,1733983953560.8545c1fe97252abe3447dcfaebd5cfa5., aborting... java.io.IOException: Failed to report opened region to master: test,,1733983953560.8545c1fe97252abe3447dcfaebd5cfa5. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-12T06:12:39,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42285 is added to blk_1073741865_1041 (size=4192) 2024-12-12T06:12:39,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42285 is added to blk_1073741834_1010 (size=6263) 2024-12-12T06:12:39,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38541 is added to blk_1073741834_1010 (size=6263) 2024-12-12T06:12:39,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43493 is added to blk_1073741834_1010 (size=6263) 2024-12-12T06:12:39,779 ERROR [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2440): ***** ABORTING region server ffbfd3107920,46199,1733983951048: Failed to open region test,,1733983953560.8545c1fe97252abe3447dcfaebd5cfa5. and can not recover ***** java.io.IOException: Failed to report opened region to master: test,,1733983953560.8545c1fe97252abe3447dcfaebd5cfa5. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-12T06:12:39,779 ERROR [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-12T06:12:39,779 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-12T06:12:39,779 WARN [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,ppp,1733983953560.1202ee0cad3e428893fcdbc630d4f9b9., aborting... java.io.IOException: Failed to report opened region to master: test,ppp,1733983953560.1202ee0cad3e428893fcdbc630d4f9b9. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-12T06:12:39,780 ERROR [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2440): ***** ABORTING region server ffbfd3107920,42275,1733983951224: Failed to open region test,ppp,1733983953560.1202ee0cad3e428893fcdbc630d4f9b9. and can not recover ***** java.io.IOException: Failed to report opened region to master: test,ppp,1733983953560.1202ee0cad3e428893fcdbc630d4f9b9. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-12T06:12:39,780 ERROR [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-12T06:12:39,780 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-12T06:12:39,780 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-12T06:12:39,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38541 is added to blk_1073741865_1041 (size=4192) 2024-12-12T06:12:39,780 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-12T06:12:39,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43493 is added to blk_1073741865_1041 (size=4192) 2024-12-12T06:12:39,782 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-12T06:12:39,783 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-12T06:12:39,783 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-12T06:12:39,784 DEBUG [RS:0;ffbfd3107920:46199 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/oldWALs 2024-12-12T06:12:39,784 INFO [RS:0;ffbfd3107920:46199 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ffbfd3107920%2C46199%2C1733983951048:(num 1733983952728) 2024-12-12T06:12:39,784 DEBUG [RS:0;ffbfd3107920:46199 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-12T06:12:39,784 INFO [RS:0;ffbfd3107920:46199 {}] regionserver.LeaseManager(133): Closed leases 2024-12-12T06:12:39,784 INFO [RS:0;ffbfd3107920:46199 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-12T06:12:39,784 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-12T06:12:39,784 INFO [RS:0;ffbfd3107920:46199 {}] hbase.ChoreService(370): Chore service for: regionserver/ffbfd3107920:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-12T06:12:39,784 INFO [RS:0;ffbfd3107920:46199 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-12T06:12:39,785 INFO [RS:0;ffbfd3107920:46199 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-12T06:12:39,785 INFO [regionserver/ffbfd3107920:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-12T06:12:39,785 INFO [RS:0;ffbfd3107920:46199 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-12T06:12:39,785 INFO [RS:0;ffbfd3107920:46199 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-12T06:12:39,787 INFO [RS:0;ffbfd3107920:46199 {}] executor.ExecutorService(108): Executor-9-RS_OPEN_REGION-regionserver/ffbfd3107920:0 had [Event #40 of type M_RS_OPEN_REGION (class org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler)] on shutdown 2024-12-12T06:12:39,787 INFO [RS:0;ffbfd3107920:46199 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46199 2024-12-12T06:12:39,790 DEBUG [RS:2;ffbfd3107920:42275 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/oldWALs 2024-12-12T06:12:39,791 INFO [RS:2;ffbfd3107920:42275 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ffbfd3107920%2C42275%2C1733983951224.meta:.meta(num 1733983957697) 2024-12-12T06:12:39,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42285 is added to blk_1073741833_1009 (size=6268) 2024-12-12T06:12:39,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38541 is added to blk_1073741833_1009 (size=6268) 2024-12-12T06:12:39,794 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 512522744 }, "NonHeapMemoryUsage": { "committed": 164560896, "init": 7667712, "max": -1, "used": 161924256 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "ffbfd3107920", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 2, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 39092, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 115, "ProcessCallTime_min": 0, "ProcessCallTime_max": 15, "ProcessCallTime_mean": 2, "ProcessCallTime_25th_percentile": 3, "ProcessCallTime_median": 7, "ProcessCallTime_75th_percentile": 11, "ProcessCallTime_90th_percentile": 13, "ProcessCallTime_95th_percentile": 14, "ProcessCallTime_98th_percentile": 14, "ProcessCallTime_99th_percentile": 14, "ProcessCallTime_99.9th_percentile": 14, "ProcessCallTime_TimeRangeCount_0-1": 115, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 115, "QueueCallTime_min": 0, "QueueCallTime_max": 9, "QueueCallTime_mean": 1, "QueueCallTime_25th_percentile": 2, "QueueCallTime_median": 4, "QueueCallTime_75th_percentile": 6, "QueueCallTime_90th_percentile": 8, "QueueCallTime_95th_percentile": 8, "QueueCallTime_98th_percentile": 8, "QueueCallTime_99th_percentile": 8, "QueueCallTime_99.9th_percentile": 8, "QueueCallTime_TimeRangeCount_0-1": 115, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 115, "TotalCallTime_min": 0, "TotalCallTime_max": 15, "TotalCallTime_mean": 3, "TotalCallTime_25th_percentile": 3, "TotalCallTime_median": 7, "TotalCallTime_75th_percentile": 11, "TotalCallTime_90th_percentile": 13, "TotalCallTime_95th_percentile": 14, "TotalCallTime_98th_percentile": 14, "TotalCallTime_99th_percentile": 14, "TotalCallTime_99.9th_percentile": 14, "TotalCallTime_TimeRangeCount_0-1": 115, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 115, "ResponseSize_min": 0, "ResponseSize_max": 22359, "ResponseSize_mean": 594, "ResponseSize_25th_percentile": 5589, "ResponseSize_median": 11179, "ResponseSize_75th_percentile": 16769, "ResponseSize_90th_percentile": 20123, "ResponseSize_95th_percentile": 21241, "ResponseSize_98th_percentile": 21911, "ResponseSize_99th_percentile": 22135, "ResponseSize_99.9th_percentile": 22336, "ResponseSize_SizeRangeCount_0-10": 115, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 115, "RequestSize_min": 81, "RequestSize_max": 8095, "RequestSize_mean": 307, "RequestSize_25th_percentile": 2084, "RequestSize_median": 4088, "RequestSize_75th_percentile": 6091, "RequestSize_90th_percentile": 7293, "RequestSize_95th_percentile": 7694, "RequestSize_98th_percentile": 7934, "RequestSize_99th_percentile": 8014, "RequestSize_99.9th_percentile": 8086, "RequestSize_SizeRangeCount_0-10": 115, "sentBytes": 54885 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "ffbfd3107920", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:59802", "tag.serverName": "ffbfd3107920,33455,1733983951151", "tag.clusterId": "85203218-ceb3-452a-ac4d-28616485cd2d", "tag.Context": "regionserver", "tag.Hostname": "ffbfd3107920", "regionCount": 9, "storeCount": 12, "hlogFileCount": 2, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 30111, "memStoreHeapSize": 53936, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1733983951151, "averageRegionSize": 3345, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 3337, "localBytesRead": 3337, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 18.8, "writeRequestRatePerSecond": 17.2, "ByteBuffAllocatorHeapAllocationBytes": 7466, "ByteBuffAllocatorPoolAllocationBytes": 465920, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 79, "totalRowActionRequestCount": 180, "readRequestCount": 94, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 86, "rpcGetRequestCount": 2, "rpcFullScanRequestCount": 9, "rpcScanRequestCount": 16, "rpcMultiRequestCount": 1, "rpcMutateRequestCount": 60, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 16, "ScanTime_min": 0, "ScanTime_max": 13, "ScanTime_mean": 1, "ScanTime_25th_percentile": 3, "ScanTime_median": 6, "ScanTime_75th_percentile": 9, "ScanTime_90th_percentile": 11, "ScanTime_95th_percentile": 12, "ScanTime_98th_percentile": 12, "ScanTime_99th_percentile": 12, "ScanTime_99.9th_percentile": 12, "ScanTime_TimeRangeCount_0-1": 16, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 60, "Put_min": 1, "Put_max": 7, "Put_mean": 1, "Put_25th_percentile": 2, "Put_median": 4, "Put_75th_percentile": 5, "Put_90th_percentile": 6, "Put_95th_percentile": 6, "Put_98th_percentile": 6, "Put_99th_percentile": 6, "Put_99.9th_percentile": 6, "Put_TimeRangeCount_0-1": 60, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 1, "PutBatch_min": 7, "PutBatch_max": 7, "PutBatch_mean": 7, "PutBatch_25th_percentile": 7, "PutBatch_median": 7, "PutBatch_75th_percentile": 7, "PutBatch_90th_percentile": 7, "PutBatch_95th_percentile": 7, "PutBatch_98th_percentile": 7, "PutBatch_99th_percentile": 7, "PutBatch_99.9th_percentile": 7, "PutBatch_TimeRangeCount_0-1": 1, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 2, "Get_min": 1, "Get_max": 2, "Get_mean": 1, "Get_25th_percentile": 1, "Get_median": 1, "Get_75th_percentile": 1, "Get_90th_percentile": 1, "Get_95th_percentile": 1, "Get_98th_percentile": 1, "Get_99th_percentile": 1, "Get_99.9th_percentile": 1, "Get_TimeRangeCount_0-1": 2, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 16, "ScanSize_min": 0, "ScanSize_max": 22232, "ScanSize_mean": 4186, "ScanSize_25th_percentile": 5558, "ScanSize_median": 11116, "ScanSize_75th_percentile": 16674, "ScanSize_90th_percentile": 20008, "ScanSize_95th_percentile": 21120, "ScanSize_98th_percentile": 21787, "ScanSize_99th_percentile": 22009, "ScanSize_99.9th_percentile": 22209, "ScanSize_SizeRangeCount_0-10": 16, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-12-12T06:12:39,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43493 is added to blk_1073741833_1009 (size=6268) 2024-12-12T06:12:39,795 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 512522744 }, "NonHeapMemoryUsage": { "committed": 164560896, "init": 7667712, "max": -1, "used": 161923632 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "ffbfd3107920", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 2, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 39092, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 115, "ProcessCallTime_min": 0, "ProcessCallTime_max": 15, "ProcessCallTime_mean": 2, "ProcessCallTime_25th_percentile": 3, "ProcessCallTime_median": 7, "ProcessCallTime_75th_percentile": 11, "ProcessCallTime_90th_percentile": 13, "ProcessCallTime_95th_percentile": 14, "ProcessCallTime_98th_percentile": 14, "ProcessCallTime_99th_percentile": 14, "ProcessCallTime_99.9th_percentile": 14, "ProcessCallTime_TimeRangeCount_0-1": 115, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 115, "QueueCallTime_min": 0, "QueueCallTime_max": 9, "QueueCallTime_mean": 1, "QueueCallTime_25th_percentile": 2, "QueueCallTime_median": 4, "QueueCallTime_75th_percentile": 6, "QueueCallTime_90th_percentile": 8, "QueueCallTime_95th_percentile": 8, "QueueCallTime_98th_percentile": 8, "QueueCallTime_99th_percentile": 8, "QueueCallTime_99.9th_percentile": 8, "QueueCallTime_TimeRangeCount_0-1": 115, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 115, "TotalCallTime_min": 0, "TotalCallTime_max": 15, "TotalCallTime_mean": 3, "TotalCallTime_25th_percentile": 3, "TotalCallTime_median": 7, "TotalCallTime_75th_percentile": 11, "TotalCallTime_90th_percentile": 13, "TotalCallTime_95th_percentile": 14, "TotalCallTime_98th_percentile": 14, "TotalCallTime_99th_percentile": 14, "TotalCallTime_99.9th_percentile": 14, "TotalCallTime_TimeRangeCount_0-1": 115, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 115, "ResponseSize_min": 0, "ResponseSize_max": 22359, "ResponseSize_mean": 594, "ResponseSize_25th_percentile": 5589, "ResponseSize_median": 11179, "ResponseSize_75th_percentile": 16769, "ResponseSize_90th_percentile": 20123, "ResponseSize_95th_percentile": 21241, "ResponseSize_98th_percentile": 21911, "ResponseSize_99th_percentile": 22135, "ResponseSize_99.9th_percentile": 22336, "ResponseSize_SizeRangeCount_0-10": 115, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 115, "RequestSize_min": 81, "RequestSize_max": 8095, "RequestSize_mean": 307, "RequestSize_25th_percentile": 2084, "RequestSize_median": 4088, "RequestSize_75th_percentile": 6091, "RequestSize_90th_percentile": 7293, "RequestSize_95th_percentile": 7694, "RequestSize_98th_percentile": 7934, "RequestSize_99th_percentile": 8014, "RequestSize_99.9th_percentile": 8086, "RequestSize_SizeRangeCount_0-10": 115, "sentBytes": 54885 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "ffbfd3107920", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:59802", "tag.serverName": "ffbfd3107920,33455,1733983951151", "tag.clusterId": "85203218-ceb3-452a-ac4d-28616485cd2d", "tag.Context": "regionserver", "tag.Hostname": "ffbfd3107920", "regionCount": 9, "storeCount": 12, "hlogFileCount": 2, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 30111, "memStoreHeapSize": 53936, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1733983951151, "averageRegionSize": 3345, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 3337, "localBytesRead": 3337, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 18.8, "writeRequestRatePerSecond": 17.2, "ByteBuffAllocatorHeapAllocationBytes": 7466, "ByteBuffAllocatorPoolAllocationBytes": 465920, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 79, "totalRowActionRequestCount": 180, "readRequestCount": 94, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 86, "rpcGetRequestCount": 2, "rpcFullScanRequestCount": 9, "rpcScanRequestCount": 16, "rpcMultiRequestCount": 1, "rpcMutateRequestCount": 60, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 16, "ScanTime_min": 0, "ScanTime_max": 13, "ScanTime_mean": 1, "ScanTime_25th_percentile": 3, "ScanTime_median": 6, "ScanTime_75th_percentile": 9, "ScanTime_90th_percentile": 11, "ScanTime_95th_percentile": 12, "ScanTime_98th_percentile": 12, "ScanTime_99th_percentile": 12, "ScanTime_99.9th_percentile": 12, "ScanTime_TimeRangeCount_0-1": 16, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 60, "Put_min": 1, "Put_max": 7, "Put_mean": 1, "Put_25th_percentile": 2, "Put_median": 4, "Put_75th_percentile": 5, "Put_90th_percentile": 6, "Put_95th_percentile": 6, "Put_98th_percentile": 6, "Put_99th_percentile": 6, "Put_99.9th_percentile": 6, "Put_TimeRangeCount_0-1": 60, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 1, "PutBatch_min": 7, "PutBatch_max": 7, "PutBatch_mean": 7, "PutBatch_25th_percentile": 7, "PutBatch_median": 7, "PutBatch_75th_percentile": 7, "PutBatch_90th_percentile": 7, "PutBatch_95th_percentile": 7, "PutBatch_98th_percentile": 7, "PutBatch_99th_percentile": 7, "PutBatch_99.9th_percentile": 7, "PutBatch_TimeRangeCount_0-1": 1, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 2, "Get_min": 1, "Get_max": 2, "Get_mean": 1, "Get_25th_percentile": 1, "Get_median": 1, "Get_75th_percentile": 1, "Get_90th_percentile": 1, "Get_95th_percentile": 1, "Get_98th_percentile": 1, "Get_99th_percentile": 1, "Get_99.9th_percentile": 1, "Get_TimeRangeCount_0-1": 2, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 16, "ScanSize_min": 0, "ScanSize_max": 22232, "ScanSize_mean": 4186, "ScanSize_25th_percentile": 5558, "ScanSize_median": 11116, "ScanSize_75th_percentile": 16674, "ScanSize_90th_percentile": 20008, "ScanSize_95th_percentile": 21120, "ScanSize_98th_percentile": 21787, "ScanSize_99th_percentile": 22009, "ScanSize_99.9th_percentile": 22209, "ScanSize_SizeRangeCount_0-10": 16, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-12-12T06:12:39,796 WARN [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2470): Unable to report fatal error to master org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=ffbfd3107920:45209 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:344) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportRSFatalError(RegionServerStatusProtos.java:17290) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.abort(HRegionServer.java:2467) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abortRegionServer(SingleProcessHBaseCluster.java:192) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$2.run(SingleProcessHBaseCluster.java:185) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abort(SingleProcessHBaseCluster.java:182) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.handleException(AssignRegionHandler.java:173) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:106) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=ffbfd3107920:45209 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 15 more Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 15 more 2024-12-12T06:12:39,796 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(132): Open test,yyy,1733983953560.ae428798a6433b4d48a205ab3d14ba42. 2024-12-12T06:12:39,797 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7752): Opening region: {ENCODED => ae428798a6433b4d48a205ab3d14ba42, NAME => 'test,yyy,1733983953560.ae428798a6433b4d48a205ab3d14ba42.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-12-12T06:12:39,797 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ae428798a6433b4d48a205ab3d14ba42 2024-12-12T06:12:39,797 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(898): Instantiated test,yyy,1733983953560.ae428798a6433b4d48a205ab3d14ba42.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-12T06:12:39,797 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7794): checking encryption for ae428798a6433b4d48a205ab3d14ba42 2024-12-12T06:12:39,797 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7797): checking classloading for ae428798a6433b4d48a205ab3d14ba42 2024-12-12T06:12:39,798 INFO [StoreOpener-ae428798a6433b4d48a205ab3d14ba42-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ae428798a6433b4d48a205ab3d14ba42 2024-12-12T06:12:39,800 INFO [StoreOpener-ae428798a6433b4d48a205ab3d14ba42-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ae428798a6433b4d48a205ab3d14ba42 columnFamilyName family 2024-12-12T06:12:39,800 DEBUG [StoreOpener-ae428798a6433b4d48a205ab3d14ba42-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-12T06:12:39,800 INFO [StoreOpener-ae428798a6433b4d48a205ab3d14ba42-1 {}] regionserver.HStore(327): Store=ae428798a6433b4d48a205ab3d14ba42/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-12T06:12:39,801 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1038): replaying wal for ae428798a6433b4d48a205ab3d14ba42 2024-12-12T06:12:39,801 DEBUG [RS:2;ffbfd3107920:42275 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/oldWALs 2024-12-12T06:12:39,802 INFO [RS:2;ffbfd3107920:42275 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ffbfd3107920%2C42275%2C1733983951224:(num 1733983952728) 2024-12-12T06:12:39,802 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/data/default/test/ae428798a6433b4d48a205ab3d14ba42 2024-12-12T06:12:39,802 DEBUG [RS:2;ffbfd3107920:42275 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-12T06:12:39,802 INFO [RS:2;ffbfd3107920:42275 {}] regionserver.LeaseManager(133): Closed leases 2024-12-12T06:12:39,802 INFO [RS:2;ffbfd3107920:42275 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-12T06:12:39,802 INFO [RS:2;ffbfd3107920:42275 {}] hbase.ChoreService(370): Chore service for: regionserver/ffbfd3107920:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-12T06:12:39,802 INFO [RS:2;ffbfd3107920:42275 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-12T06:12:39,802 INFO [regionserver/ffbfd3107920:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-12T06:12:39,802 INFO [RS:2;ffbfd3107920:42275 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42275 2024-12-12T06:12:39,803 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1048): stopping wal replay for ae428798a6433b4d48a205ab3d14ba42 2024-12-12T06:12:39,803 WARN [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(979): Failed initialize of region= test,yyy,1733983953560.ae428798a6433b4d48a205ab3d14ba42., starting to roll back memstore java.io.InterruptedIOException: Call interrupted at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy41.getListing(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$getListing$21(ClientNamenodeProtocolTranslatorPB.java:621) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getListing(ClientNamenodeProtocolTranslatorPB.java:621) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy42.getListing(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy45.getListing(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy45.getListing(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.listPaths(DFSClient.java:1733) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.listPaths(DFSClient.java:1717) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.listStatusInternal(DistributedFileSystem.java:1112) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.access$600(DistributedFileSystem.java:148) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$24.doCall(DistributedFileSystem.java:1187) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$24.doCall(DistributedFileSystem.java:1184) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.listStatus(DistributedFileSystem.java:1194) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystem.listStatus(FileSystem.java:2078) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystem.listStatus(FileSystem.java:2122) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.util.CommonFSUtils.listStatus(CommonFSUtils.java:602) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.wal.WALSplitUtil.getSplitEditFilesSorted(WALSplitUtil.java:301) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.replayRecoveredEditsIfAny(HRegion.java:5485) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.initializeRegionInternals(HRegion.java:1042) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.initialize(HRegion.java:976) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7799) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegionFromTableDir(HRegion.java:7754) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7729) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7688) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7644) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:143) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-12T06:12:39,805 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(6241): Drop memstore for Store family in region test,yyy,1733983953560.ae428798a6433b4d48a205ab3d14ba42. , dropped memstoresize: [dataSize=0, getHeapSize=256, getOffHeapSize=0, getCellsCount=0 } 2024-12-12T06:12:39,805 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1006): Region open journal for ae428798a6433b4d48a205ab3d14ba42: Running coprocessor pre-open hook at 1733983959797Writing region info on filesystem at 1733983959797Initializing all the Stores at 1733983959798 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733983959798Exception during region test,yyy,1733983953560.ae428798a6433b4d48a205ab3d14ba42. initialization. at 1733983959805 (+7 ms) 2024-12-12T06:12:39,805 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1722): Closing ae428798a6433b4d48a205ab3d14ba42, disabling compactions & flushes 2024-12-12T06:12:39,805 INFO [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1755): Closing region test,yyy,1733983953560.ae428798a6433b4d48a205ab3d14ba42. 2024-12-12T06:12:39,805 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1733983953560.ae428798a6433b4d48a205ab3d14ba42. 2024-12-12T06:12:39,805 WARN [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1802): Interrupted while waiting for close lock on test,yyy,1733983953560.ae428798a6433b4d48a205ab3d14ba42. java.lang.InterruptedException: null at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireNanos(AbstractQueuedSynchronizer.java:993) ~[?:?] at java.util.concurrent.locks.ReentrantReadWriteLock$WriteLock.tryLock(ReentrantReadWriteLock.java:1131) ~[?:?] at org.apache.hadoop.hbase.regionserver.HRegion.doClose(HRegion.java:1794) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.close(HRegion.java:1672) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.close(HRegion.java:1627) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.close(HRegion.java:1610) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7817) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegionFromTableDir(HRegion.java:7754) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7729) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7688) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7644) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:143) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-12T06:12:39,805 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1676): Region close journal for ae428798a6433b4d48a205ab3d14ba42: Waiting for close lock at 1733983959805Running coprocessor pre-close hooks at 1733983959805Disabling compacts and flushes for region at 1733983959805 2024-12-12T06:12:39,805 WARN [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7819): Open region: {ENCODED => ae428798a6433b4d48a205ab3d14ba42, NAME => 'test,yyy,1733983953560.ae428798a6433b4d48a205ab3d14ba42.', STARTKEY => 'yyy', ENDKEY => 'zzz'} failed. Try close region but got exception java.io.InterruptedIOException: Interrupted while waiting for close lock on test,yyy,1733983953560.ae428798a6433b4d48a205ab3d14ba42. at org.apache.hadoop.hbase.regionserver.HRegion.doClose(HRegion.java:1803) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.close(HRegion.java:1672) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.close(HRegion.java:1627) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.close(HRegion.java:1610) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7817) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegionFromTableDir(HRegion.java:7754) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7729) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7688) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7644) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:143) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireNanos(AbstractQueuedSynchronizer.java:993) ~[?:?] at java.util.concurrent.locks.ReentrantReadWriteLock$WriteLock.tryLock(ReentrantReadWriteLock.java:1131) ~[?:?] at org.apache.hadoop.hbase.regionserver.HRegion.doClose(HRegion.java:1794) ~[classes/:?] ... 13 more 2024-12-12T06:12:39,806 WARN [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(84): Failed to open region test,yyy,1733983953560.ae428798a6433b4d48a205ab3d14ba42., will report to master java.io.InterruptedIOException: Call interrupted at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy41.getListing(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$getListing$21(ClientNamenodeProtocolTranslatorPB.java:621) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getListing(ClientNamenodeProtocolTranslatorPB.java:621) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy42.getListing(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy45.getListing(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy45.getListing(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.listPaths(DFSClient.java:1733) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.listPaths(DFSClient.java:1717) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.listStatusInternal(DistributedFileSystem.java:1112) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.access$600(DistributedFileSystem.java:148) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$24.doCall(DistributedFileSystem.java:1187) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$24.doCall(DistributedFileSystem.java:1184) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.listStatus(DistributedFileSystem.java:1194) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystem.listStatus(FileSystem.java:2078) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystem.listStatus(FileSystem.java:2122) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.util.CommonFSUtils.listStatus(CommonFSUtils.java:602) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.wal.WALSplitUtil.getSplitEditFilesSorted(WALSplitUtil.java:301) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.replayRecoveredEditsIfAny(HRegion.java:5485) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.initializeRegionInternals(HRegion.java:1042) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.initialize(HRegion.java:976) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7799) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegionFromTableDir(HRegion.java:7754) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7729) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7688) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7644) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:143) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-12T06:12:39,806 WARN [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,yyy,1733983953560.ae428798a6433b4d48a205ab3d14ba42., aborting... java.io.IOException: Failed to report failed open to master: test,yyy,1733983953560.ae428798a6433b4d48a205ab3d14ba42. at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.cleanUpAndReportFailure(AssignRegionHandler.java:93) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:145) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-12T06:12:39,807 DEBUG [RS_OPEN_REGION-regionserver/ffbfd3107920:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2435): Abort already in progress. Ignoring the current request with reason: Failed to open region test,yyy,1733983953560.ae428798a6433b4d48a205ab3d14ba42. and can not recover 2024-12-12T06:12:39,817 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46199-0x10018b511340001, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ffbfd3107920,46199,1733983951048 2024-12-12T06:12:39,817 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45209-0x10018b511340000, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-12T06:12:39,818 INFO [RS:0;ffbfd3107920:46199 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-12T06:12:39,826 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42275-0x10018b511340003, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ffbfd3107920,42275,1733983951224 2024-12-12T06:12:39,826 INFO [RS:2;ffbfd3107920:42275 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-12T06:12:39,834 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ffbfd3107920,46199,1733983951048] 2024-12-12T06:12:39,851 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ffbfd3107920,46199,1733983951048 already deleted, retry=false 2024-12-12T06:12:39,851 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; ffbfd3107920,46199,1733983951048 expired; onlineServers=1 2024-12-12T06:12:39,851 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ffbfd3107920,42275,1733983951224] 2024-12-12T06:12:39,859 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ffbfd3107920,42275,1733983951224 already deleted, retry=false 2024-12-12T06:12:39,859 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; ffbfd3107920,42275,1733983951224 expired; onlineServers=0 2024-12-12T06:12:39,859 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'ffbfd3107920,45209,1733983950336' ***** 2024-12-12T06:12:39,859 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-12T06:12:39,860 INFO [M:0;ffbfd3107920:45209 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-12T06:12:39,860 INFO [M:0;ffbfd3107920:45209 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-12T06:12:39,860 DEBUG [M:0;ffbfd3107920:45209 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-12T06:12:39,860 DEBUG [M:0;ffbfd3107920:45209 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-12T06:12:39,860 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-12T06:12:39,860 DEBUG [master/ffbfd3107920:0:becomeActiveMaster-HFileCleaner.large.0-1733983952331 {}] cleaner.HFileCleaner(306): Exit Thread[master/ffbfd3107920:0:becomeActiveMaster-HFileCleaner.large.0-1733983952331,5,FailOnTimeoutGroup] 2024-12-12T06:12:39,860 DEBUG [master/ffbfd3107920:0:becomeActiveMaster-HFileCleaner.small.0-1733983952332 {}] cleaner.HFileCleaner(306): Exit Thread[master/ffbfd3107920:0:becomeActiveMaster-HFileCleaner.small.0-1733983952332,5,FailOnTimeoutGroup] 2024-12-12T06:12:39,861 INFO [M:0;ffbfd3107920:45209 {}] hbase.ChoreService(370): Chore service for: master/ffbfd3107920:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-12T06:12:39,861 INFO [M:0;ffbfd3107920:45209 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-12T06:12:39,861 DEBUG [M:0;ffbfd3107920:45209 {}] master.HMaster(1795): Stopping service threads 2024-12-12T06:12:39,861 INFO [M:0;ffbfd3107920:45209 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-12T06:12:39,861 INFO [M:0;ffbfd3107920:45209 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-12T06:12:39,862 INFO [M:0;ffbfd3107920:45209 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-12T06:12:39,862 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-12T06:12:39,872 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45209-0x10018b511340000, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-12T06:12:39,872 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45209-0x10018b511340000, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-12T06:12:39,873 DEBUG [M:0;ffbfd3107920:45209 {}] zookeeper.ZKUtil(347): master:45209-0x10018b511340000, quorum=127.0.0.1:59802, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-12T06:12:39,873 WARN [M:0;ffbfd3107920:45209 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-12T06:12:39,874 INFO [M:0;ffbfd3107920:45209 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/.lastflushedseqids 2024-12-12T06:12:39,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38541 is added to blk_1073741870_1046 (size=1) 2024-12-12T06:12:39,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43493 is added to blk_1073741870_1046 (size=1) 2024-12-12T06:12:39,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42285 is added to blk_1073741870_1046 (size=1) 2024-12-12T06:12:39,892 INFO [M:0;ffbfd3107920:45209 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-12T06:12:39,892 INFO [M:0;ffbfd3107920:45209 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-12T06:12:39,892 DEBUG [M:0;ffbfd3107920:45209 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-12T06:12:39,892 INFO [M:0;ffbfd3107920:45209 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-12T06:12:39,892 DEBUG [M:0;ffbfd3107920:45209 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-12T06:12:39,892 DEBUG [M:0;ffbfd3107920:45209 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-12T06:12:39,892 DEBUG [M:0;ffbfd3107920:45209 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-12T06:12:39,892 INFO [M:0;ffbfd3107920:45209 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=259.49 KB heapSize=315.40 KB 2024-12-12T06:12:39,908 DEBUG [M:0;ffbfd3107920:45209 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4575b5b2a00e4692b67b792798ef50ee is 82, key is hbase:meta,,1/info:regioninfo/1733983957994/Put/seqid=0 2024-12-12T06:12:39,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42285 is added to blk_1073741871_1047 (size=6071) 2024-12-12T06:12:39,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43493 is added to blk_1073741871_1047 (size=6071) 2024-12-12T06:12:39,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38541 is added to blk_1073741871_1047 (size=6071) 2024-12-12T06:12:39,915 INFO [M:0;ffbfd3107920:45209 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=599 (bloomFilter=true), to=hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4575b5b2a00e4692b67b792798ef50ee 2024-12-12T06:12:39,934 INFO [RS:0;ffbfd3107920:46199 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-12T06:12:39,934 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46199-0x10018b511340001, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-12T06:12:39,934 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46199-0x10018b511340001, quorum=127.0.0.1:59802, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-12T06:12:39,934 INFO [RS:0;ffbfd3107920:46199 {}] regionserver.HRegionServer(1031): Exiting; stopping=ffbfd3107920,46199,1733983951048; zookeeper connection closed. 2024-12-12T06:12:39,934 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@43adcc8e {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@43adcc8e 2024-12-12T06:12:39,939 DEBUG [M:0;ffbfd3107920:45209 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38031/user/jenkins/test-data/b3c618ed-4309-866d-0bb0-fd8f03c1e6b0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ade098e5a73541fb951048ca9beb6018 is 2987, key is \x00\x00\x00\x00\x00\x00\x00