2024-11-14 14:40:54,492 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-14 14:40:54,514 main DEBUG Took 0.019391 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-14 14:40:54,515 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-14 14:40:54,515 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-14 14:40:54,517 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-14 14:40:54,520 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 14:40:54,531 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-14 14:40:54,551 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:40:54,553 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 14:40:54,554 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:40:54,554 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 14:40:54,555 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:40:54,555 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 14:40:54,557 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:40:54,557 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 14:40:54,557 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:40:54,558 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 14:40:54,559 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:40:54,559 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 14:40:54,560 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:40:54,560 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 14:40:54,561 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:40:54,561 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 14:40:54,562 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:40:54,562 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 14:40:54,563 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:40:54,563 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 14:40:54,564 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:40:54,564 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 14:40:54,565 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:40:54,565 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 14:40:54,566 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:40:54,566 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-14 14:40:54,568 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:40:54,570 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-14 14:40:54,573 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-14 14:40:54,573 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-14 14:40:54,583 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-14 14:40:54,584 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-14 14:40:54,597 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-14 14:40:54,601 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-14 14:40:54,604 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-14 14:40:54,606 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-14 14:40:54,607 main DEBUG createAppenders(={Console}) 2024-11-14 14:40:54,608 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 initialized 2024-11-14 14:40:54,608 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-14 14:40:54,609 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 OK. 2024-11-14 14:40:54,609 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-14 14:40:54,610 main DEBUG OutputStream closed 2024-11-14 14:40:54,610 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-14 14:40:54,610 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-14 14:40:54,611 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@c88a337 OK 2024-11-14 14:40:54,819 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-14 14:40:54,830 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-14 14:40:54,831 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-14 14:40:54,836 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-14 14:40:54,838 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-14 14:40:54,839 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-14 14:40:54,839 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-14 14:40:54,841 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-14 14:40:54,842 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-14 14:40:54,842 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-14 14:40:54,843 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-14 14:40:54,843 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-14 14:40:54,844 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-14 14:40:54,845 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-14 14:40:54,845 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-14 14:40:54,846 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-14 14:40:54,846 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-14 14:40:54,847 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-14 14:40:54,853 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-14 14:40:54,854 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@13330ac6) with optional ClassLoader: null 2024-11-14 14:40:54,854 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-14 14:40:54,856 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@13330ac6] started OK. 2024-11-14T14:40:54,878 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP timeout: 13 mins 2024-11-14 14:40:54,883 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-14 14:40:54,884 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-14T14:40:55,393 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/694aaacd-0466-2f0d-621d-715bf1fdc22e 2024-11-14T14:40:55,417 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$HMasterForTest, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-14T14:40:55,435 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/694aaacd-0466-2f0d-621d-715bf1fdc22e/cluster_9a8ac74c-702c-79fa-009c-bd4fec459006, deleteOnExit=true 2024-11-14T14:40:55,436 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-14T14:40:55,437 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/694aaacd-0466-2f0d-621d-715bf1fdc22e/test.cache.data in system properties and HBase conf 2024-11-14T14:40:55,437 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/694aaacd-0466-2f0d-621d-715bf1fdc22e/hadoop.tmp.dir in system properties and HBase conf 2024-11-14T14:40:55,438 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/694aaacd-0466-2f0d-621d-715bf1fdc22e/hadoop.log.dir in system properties and HBase conf 2024-11-14T14:40:55,439 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/694aaacd-0466-2f0d-621d-715bf1fdc22e/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-14T14:40:55,439 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/694aaacd-0466-2f0d-621d-715bf1fdc22e/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-14T14:40:55,439 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-14T14:40:55,560 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-14T14:40:55,725 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T14:40:55,734 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/694aaacd-0466-2f0d-621d-715bf1fdc22e/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-14T14:40:55,735 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/694aaacd-0466-2f0d-621d-715bf1fdc22e/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-14T14:40:55,737 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/694aaacd-0466-2f0d-621d-715bf1fdc22e/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-14T14:40:55,738 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/694aaacd-0466-2f0d-621d-715bf1fdc22e/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-14T14:40:55,739 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/694aaacd-0466-2f0d-621d-715bf1fdc22e/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-14T14:40:55,741 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/694aaacd-0466-2f0d-621d-715bf1fdc22e/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-14T14:40:55,747 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/694aaacd-0466-2f0d-621d-715bf1fdc22e/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-14T14:40:55,749 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/694aaacd-0466-2f0d-621d-715bf1fdc22e/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-14T14:40:55,755 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/694aaacd-0466-2f0d-621d-715bf1fdc22e/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-14T14:40:55,764 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/694aaacd-0466-2f0d-621d-715bf1fdc22e/nfs.dump.dir in system properties and HBase conf 2024-11-14T14:40:55,765 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/694aaacd-0466-2f0d-621d-715bf1fdc22e/java.io.tmpdir in system properties and HBase conf 2024-11-14T14:40:55,766 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/694aaacd-0466-2f0d-621d-715bf1fdc22e/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-14T14:40:55,773 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/694aaacd-0466-2f0d-621d-715bf1fdc22e/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-14T14:40:55,774 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/694aaacd-0466-2f0d-621d-715bf1fdc22e/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-14T14:40:57,065 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-14T14:40:57,180 INFO [Time-limited test {}] log.Log(170): Logging initialized @3722ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-14T14:40:57,280 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T14:40:57,368 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T14:40:57,430 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T14:40:57,431 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T14:40:57,432 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-14T14:40:57,454 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T14:40:57,459 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3912242{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/694aaacd-0466-2f0d-621d-715bf1fdc22e/hadoop.log.dir/,AVAILABLE} 2024-11-14T14:40:57,461 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@595320c8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T14:40:57,802 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@64418798{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/694aaacd-0466-2f0d-621d-715bf1fdc22e/java.io.tmpdir/jetty-localhost-33427-hadoop-hdfs-3_4_1-tests_jar-_-any-5287950538534158815/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-14T14:40:57,811 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@11744524{HTTP/1.1, (http/1.1)}{localhost:33427} 2024-11-14T14:40:57,812 INFO [Time-limited test {}] server.Server(415): Started @4355ms 2024-11-14T14:40:58,326 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T14:40:58,336 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T14:40:58,340 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T14:40:58,341 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T14:40:58,341 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-14T14:40:58,343 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@51c39823{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/694aaacd-0466-2f0d-621d-715bf1fdc22e/hadoop.log.dir/,AVAILABLE} 2024-11-14T14:40:58,343 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@c7135d9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T14:40:58,488 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@65e17246{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/694aaacd-0466-2f0d-621d-715bf1fdc22e/java.io.tmpdir/jetty-localhost-33867-hadoop-hdfs-3_4_1-tests_jar-_-any-7374717237148326576/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T14:40:58,490 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@220c8d80{HTTP/1.1, (http/1.1)}{localhost:33867} 2024-11-14T14:40:58,490 INFO [Time-limited test {}] server.Server(415): Started @5034ms 2024-11-14T14:40:58,565 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-14T14:40:58,763 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T14:40:58,770 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T14:40:58,783 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T14:40:58,783 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T14:40:58,783 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-14T14:40:58,784 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2ff36c4c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/694aaacd-0466-2f0d-621d-715bf1fdc22e/hadoop.log.dir/,AVAILABLE} 2024-11-14T14:40:58,785 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6331ed3f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T14:40:58,957 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@395ff33a{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/694aaacd-0466-2f0d-621d-715bf1fdc22e/java.io.tmpdir/jetty-localhost-37651-hadoop-hdfs-3_4_1-tests_jar-_-any-1256902422516518830/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T14:40:58,958 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4d24ac65{HTTP/1.1, (http/1.1)}{localhost:37651} 2024-11-14T14:40:58,958 INFO [Time-limited test {}] server.Server(415): Started @5502ms 2024-11-14T14:40:58,962 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-14T14:40:59,024 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T14:40:59,029 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T14:40:59,030 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T14:40:59,031 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T14:40:59,031 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-14T14:40:59,032 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6ed43cd1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/694aaacd-0466-2f0d-621d-715bf1fdc22e/hadoop.log.dir/,AVAILABLE} 2024-11-14T14:40:59,033 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@48a2d1c6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T14:40:59,189 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@46041a9d{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/694aaacd-0466-2f0d-621d-715bf1fdc22e/java.io.tmpdir/jetty-localhost-37987-hadoop-hdfs-3_4_1-tests_jar-_-any-13302823183028891817/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T14:40:59,190 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2a39ea25{HTTP/1.1, (http/1.1)}{localhost:37987} 2024-11-14T14:40:59,191 INFO [Time-limited test {}] server.Server(415): Started @5734ms 2024-11-14T14:40:59,194 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-14T14:40:59,206 WARN [Thread-110 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/694aaacd-0466-2f0d-621d-715bf1fdc22e/cluster_9a8ac74c-702c-79fa-009c-bd4fec459006/data/data1/current/BP-986035353-172.17.0.2-1731595256759/current, will proceed with Du for space computation calculation, 2024-11-14T14:40:59,206 WARN [Thread-108 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/694aaacd-0466-2f0d-621d-715bf1fdc22e/cluster_9a8ac74c-702c-79fa-009c-bd4fec459006/data/data3/current/BP-986035353-172.17.0.2-1731595256759/current, will proceed with Du for space computation calculation, 2024-11-14T14:40:59,206 WARN [Thread-109 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/694aaacd-0466-2f0d-621d-715bf1fdc22e/cluster_9a8ac74c-702c-79fa-009c-bd4fec459006/data/data4/current/BP-986035353-172.17.0.2-1731595256759/current, will proceed with Du for space computation calculation, 2024-11-14T14:40:59,206 WARN [Thread-111 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/694aaacd-0466-2f0d-621d-715bf1fdc22e/cluster_9a8ac74c-702c-79fa-009c-bd4fec459006/data/data2/current/BP-986035353-172.17.0.2-1731595256759/current, will proceed with Du for space computation calculation, 2024-11-14T14:40:59,262 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-14T14:40:59,262 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-14T14:40:59,338 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x92cc47aac912ee8 with lease ID 0x8d132f9ba53eb200: Processing first storage report for DS-0f3bbfa7-353f-42f1-ac96-37bfae4542a6 from datanode DatanodeRegistration(127.0.0.1:36849, datanodeUuid=0c2cd0b9-47f4-4235-8c22-bbadfd149574, infoPort=44937, infoSecurePort=0, ipcPort=40541, storageInfo=lv=-57;cid=testClusterID;nsid=522810691;c=1731595256759) 2024-11-14T14:40:59,339 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x92cc47aac912ee8 with lease ID 0x8d132f9ba53eb200: from storage DS-0f3bbfa7-353f-42f1-ac96-37bfae4542a6 node DatanodeRegistration(127.0.0.1:36849, datanodeUuid=0c2cd0b9-47f4-4235-8c22-bbadfd149574, infoPort=44937, infoSecurePort=0, ipcPort=40541, storageInfo=lv=-57;cid=testClusterID;nsid=522810691;c=1731595256759), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-14T14:40:59,340 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf35aed86eaa115d0 with lease ID 0x8d132f9ba53eb1ff: Processing first storage report for DS-728ae2cb-a461-4860-83b4-16a012c6b9ad from datanode DatanodeRegistration(127.0.0.1:45021, datanodeUuid=967fa7ac-b139-4ef6-9710-d38aa72e25ab, infoPort=34623, infoSecurePort=0, ipcPort=46757, storageInfo=lv=-57;cid=testClusterID;nsid=522810691;c=1731595256759) 2024-11-14T14:40:59,340 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf35aed86eaa115d0 with lease ID 0x8d132f9ba53eb1ff: from storage DS-728ae2cb-a461-4860-83b4-16a012c6b9ad node DatanodeRegistration(127.0.0.1:45021, datanodeUuid=967fa7ac-b139-4ef6-9710-d38aa72e25ab, infoPort=34623, infoSecurePort=0, ipcPort=46757, storageInfo=lv=-57;cid=testClusterID;nsid=522810691;c=1731595256759), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T14:40:59,340 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x92cc47aac912ee8 with lease ID 0x8d132f9ba53eb200: Processing first storage report for DS-221e0036-b2d4-4b1a-9c1d-25c591789e87 from datanode DatanodeRegistration(127.0.0.1:36849, datanodeUuid=0c2cd0b9-47f4-4235-8c22-bbadfd149574, infoPort=44937, infoSecurePort=0, ipcPort=40541, storageInfo=lv=-57;cid=testClusterID;nsid=522810691;c=1731595256759) 2024-11-14T14:40:59,340 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x92cc47aac912ee8 with lease ID 0x8d132f9ba53eb200: from storage DS-221e0036-b2d4-4b1a-9c1d-25c591789e87 node DatanodeRegistration(127.0.0.1:36849, datanodeUuid=0c2cd0b9-47f4-4235-8c22-bbadfd149574, infoPort=44937, infoSecurePort=0, ipcPort=40541, storageInfo=lv=-57;cid=testClusterID;nsid=522810691;c=1731595256759), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T14:40:59,340 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf35aed86eaa115d0 with lease ID 0x8d132f9ba53eb1ff: Processing first storage report for DS-48eacaee-193b-4f87-bb5e-7049985224e4 from datanode DatanodeRegistration(127.0.0.1:45021, datanodeUuid=967fa7ac-b139-4ef6-9710-d38aa72e25ab, infoPort=34623, infoSecurePort=0, ipcPort=46757, storageInfo=lv=-57;cid=testClusterID;nsid=522810691;c=1731595256759) 2024-11-14T14:40:59,341 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf35aed86eaa115d0 with lease ID 0x8d132f9ba53eb1ff: from storage DS-48eacaee-193b-4f87-bb5e-7049985224e4 node DatanodeRegistration(127.0.0.1:45021, datanodeUuid=967fa7ac-b139-4ef6-9710-d38aa72e25ab, infoPort=34623, infoSecurePort=0, ipcPort=46757, storageInfo=lv=-57;cid=testClusterID;nsid=522810691;c=1731595256759), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T14:40:59,417 WARN [Thread-139 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/694aaacd-0466-2f0d-621d-715bf1fdc22e/cluster_9a8ac74c-702c-79fa-009c-bd4fec459006/data/data5/current/BP-986035353-172.17.0.2-1731595256759/current, will proceed with Du for space computation calculation, 2024-11-14T14:40:59,417 WARN [Thread-140 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/694aaacd-0466-2f0d-621d-715bf1fdc22e/cluster_9a8ac74c-702c-79fa-009c-bd4fec459006/data/data6/current/BP-986035353-172.17.0.2-1731595256759/current, will proceed with Du for space computation calculation, 2024-11-14T14:40:59,443 WARN [Thread-119 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-14T14:40:59,449 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x95c842930bce3cfb with lease ID 0x8d132f9ba53eb201: Processing first storage report for DS-d2124e34-cef4-477f-b806-d1dbb55af1e5 from datanode DatanodeRegistration(127.0.0.1:45825, datanodeUuid=d7d7a462-f4b1-406b-b08e-a0cbd03c121e, infoPort=46259, infoSecurePort=0, ipcPort=33327, storageInfo=lv=-57;cid=testClusterID;nsid=522810691;c=1731595256759) 2024-11-14T14:40:59,449 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x95c842930bce3cfb with lease ID 0x8d132f9ba53eb201: from storage DS-d2124e34-cef4-477f-b806-d1dbb55af1e5 node DatanodeRegistration(127.0.0.1:45825, datanodeUuid=d7d7a462-f4b1-406b-b08e-a0cbd03c121e, infoPort=46259, infoSecurePort=0, ipcPort=33327, storageInfo=lv=-57;cid=testClusterID;nsid=522810691;c=1731595256759), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-14T14:40:59,450 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x95c842930bce3cfb with lease ID 0x8d132f9ba53eb201: Processing first storage report for DS-0354da61-c0b4-4d3f-98a0-97b7ee2e2bd2 from datanode DatanodeRegistration(127.0.0.1:45825, datanodeUuid=d7d7a462-f4b1-406b-b08e-a0cbd03c121e, infoPort=46259, infoSecurePort=0, ipcPort=33327, storageInfo=lv=-57;cid=testClusterID;nsid=522810691;c=1731595256759) 2024-11-14T14:40:59,450 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x95c842930bce3cfb with lease ID 0x8d132f9ba53eb201: from storage DS-0354da61-c0b4-4d3f-98a0-97b7ee2e2bd2 node DatanodeRegistration(127.0.0.1:45825, datanodeUuid=d7d7a462-f4b1-406b-b08e-a0cbd03c121e, infoPort=46259, infoSecurePort=0, ipcPort=33327, storageInfo=lv=-57;cid=testClusterID;nsid=522810691;c=1731595256759), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-14T14:40:59,601 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/694aaacd-0466-2f0d-621d-715bf1fdc22e 2024-11-14T14:40:59,679 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/694aaacd-0466-2f0d-621d-715bf1fdc22e/cluster_9a8ac74c-702c-79fa-009c-bd4fec459006/zookeeper_0, clientPort=60650, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/694aaacd-0466-2f0d-621d-715bf1fdc22e/cluster_9a8ac74c-702c-79fa-009c-bd4fec459006/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/694aaacd-0466-2f0d-621d-715bf1fdc22e/cluster_9a8ac74c-702c-79fa-009c-bd4fec459006/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-14T14:40:59,690 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60650 2024-11-14T14:40:59,703 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:40:59,706 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:40:59,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45021 is added to blk_1073741825_1001 (size=7) 2024-11-14T14:40:59,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45825 is added to blk_1073741825_1001 (size=7) 2024-11-14T14:40:59,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36849 is added to blk_1073741825_1001 (size=7) 2024-11-14T14:41:00,350 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b with version=8 2024-11-14T14:41:00,350 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/hbase-staging 2024-11-14T14:41:00,448 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-14T14:41:00,698 INFO [Time-limited test {}] client.ConnectionUtils(128): master/da1c6afcd1f9:0 server-side Connection retries=45 2024-11-14T14:41:00,709 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T14:41:00,710 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T14:41:00,714 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T14:41:00,714 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T14:41:00,714 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T14:41:00,861 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-14T14:41:00,929 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-14T14:41:00,938 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-14T14:41:00,942 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T14:41:00,970 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 2769 (auto-detected) 2024-11-14T14:41:00,971 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-14T14:41:00,989 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34057 2024-11-14T14:41:01,011 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:34057 connecting to ZooKeeper ensemble=127.0.0.1:60650 2024-11-14T14:41:01,043 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:340570x0, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T14:41:01,046 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:34057-0x101a73a44750000 connected 2024-11-14T14:41:01,073 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:41:01,076 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:41:01,087 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34057-0x101a73a44750000, quorum=127.0.0.1:60650, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T14:41:01,091 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b, hbase.cluster.distributed=false 2024-11-14T14:41:01,116 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34057-0x101a73a44750000, quorum=127.0.0.1:60650, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T14:41:01,121 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34057 2024-11-14T14:41:01,122 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34057 2024-11-14T14:41:01,124 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34057 2024-11-14T14:41:01,125 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34057 2024-11-14T14:41:01,127 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34057 2024-11-14T14:41:01,248 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/da1c6afcd1f9:0 server-side Connection retries=45 2024-11-14T14:41:01,250 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T14:41:01,250 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T14:41:01,250 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T14:41:01,250 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T14:41:01,251 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T14:41:01,254 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-14T14:41:01,256 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T14:41:01,257 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40103 2024-11-14T14:41:01,259 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40103 connecting to ZooKeeper ensemble=127.0.0.1:60650 2024-11-14T14:41:01,260 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:41:01,264 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:41:01,270 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:401030x0, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T14:41:01,271 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40103-0x101a73a44750001 connected 2024-11-14T14:41:01,271 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40103-0x101a73a44750001, quorum=127.0.0.1:60650, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T14:41:01,275 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-14T14:41:01,284 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-14T14:41:01,288 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40103-0x101a73a44750001, quorum=127.0.0.1:60650, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-14T14:41:01,294 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40103-0x101a73a44750001, quorum=127.0.0.1:60650, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T14:41:01,296 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40103 2024-11-14T14:41:01,298 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40103 2024-11-14T14:41:01,299 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40103 2024-11-14T14:41:01,300 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40103 2024-11-14T14:41:01,304 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40103 2024-11-14T14:41:01,322 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/da1c6afcd1f9:0 server-side Connection retries=45 2024-11-14T14:41:01,322 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T14:41:01,322 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T14:41:01,323 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T14:41:01,323 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T14:41:01,323 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T14:41:01,323 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-14T14:41:01,324 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T14:41:01,325 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46729 2024-11-14T14:41:01,326 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46729 connecting to ZooKeeper ensemble=127.0.0.1:60650 2024-11-14T14:41:01,328 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:41:01,330 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:41:01,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:467290x0, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T14:41:01,339 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:467290x0, quorum=127.0.0.1:60650, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T14:41:01,339 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46729-0x101a73a44750002 connected 2024-11-14T14:41:01,339 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-14T14:41:01,340 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-14T14:41:01,341 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46729-0x101a73a44750002, quorum=127.0.0.1:60650, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-14T14:41:01,343 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46729-0x101a73a44750002, quorum=127.0.0.1:60650, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T14:41:01,344 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46729 2024-11-14T14:41:01,344 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46729 2024-11-14T14:41:01,345 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46729 2024-11-14T14:41:01,345 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46729 2024-11-14T14:41:01,346 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46729 2024-11-14T14:41:01,364 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/da1c6afcd1f9:0 server-side Connection retries=45 2024-11-14T14:41:01,364 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T14:41:01,364 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T14:41:01,364 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T14:41:01,364 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T14:41:01,365 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T14:41:01,365 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-14T14:41:01,365 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T14:41:01,366 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46229 2024-11-14T14:41:01,367 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46229 connecting to ZooKeeper ensemble=127.0.0.1:60650 2024-11-14T14:41:01,369 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:41:01,371 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:41:01,376 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:462290x0, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T14:41:01,377 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46229-0x101a73a44750003 connected 2024-11-14T14:41:01,377 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46229-0x101a73a44750003, quorum=127.0.0.1:60650, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T14:41:01,378 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-14T14:41:01,378 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-14T14:41:01,380 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46229-0x101a73a44750003, quorum=127.0.0.1:60650, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-14T14:41:01,381 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46229-0x101a73a44750003, quorum=127.0.0.1:60650, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T14:41:01,382 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46229 2024-11-14T14:41:01,384 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46229 2024-11-14T14:41:01,385 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46229 2024-11-14T14:41:01,386 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46229 2024-11-14T14:41:01,386 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46229 2024-11-14T14:41:01,403 DEBUG [M:0;da1c6afcd1f9:34057 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;da1c6afcd1f9:34057 2024-11-14T14:41:01,404 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/da1c6afcd1f9,34057,1731595260501 2024-11-14T14:41:01,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46229-0x101a73a44750003, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T14:41:01,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40103-0x101a73a44750001, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T14:41:01,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34057-0x101a73a44750000, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T14:41:01,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46729-0x101a73a44750002, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T14:41:01,413 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34057-0x101a73a44750000, quorum=127.0.0.1:60650, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/da1c6afcd1f9,34057,1731595260501 2024-11-14T14:41:01,437 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46229-0x101a73a44750003, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-14T14:41:01,437 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40103-0x101a73a44750001, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-14T14:41:01,437 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46729-0x101a73a44750002, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-14T14:41:01,437 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40103-0x101a73a44750001, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:41:01,437 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46729-0x101a73a44750002, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:41:01,437 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34057-0x101a73a44750000, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:41:01,438 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46229-0x101a73a44750003, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:41:01,439 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34057-0x101a73a44750000, quorum=127.0.0.1:60650, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-14T14:41:01,441 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/da1c6afcd1f9,34057,1731595260501 from backup master directory 2024-11-14T14:41:01,444 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40103-0x101a73a44750001, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T14:41:01,444 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46229-0x101a73a44750003, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T14:41:01,444 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46729-0x101a73a44750002, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T14:41:01,444 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34057-0x101a73a44750000, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/da1c6afcd1f9,34057,1731595260501 2024-11-14T14:41:01,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34057-0x101a73a44750000, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T14:41:01,446 WARN [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T14:41:01,447 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=da1c6afcd1f9,34057,1731595260501 2024-11-14T14:41:01,450 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-14T14:41:01,452 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-14T14:41:01,516 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/hbase.id] with ID: cbb35874-c53a-4b8c-89da-bcc2cb2e4ea1 2024-11-14T14:41:01,516 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/.tmp/hbase.id 2024-11-14T14:41:01,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36849 is added to blk_1073741826_1002 (size=42) 2024-11-14T14:41:01,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45825 is added to blk_1073741826_1002 (size=42) 2024-11-14T14:41:01,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45021 is added to blk_1073741826_1002 (size=42) 2024-11-14T14:41:01,533 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/.tmp/hbase.id]:[hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/hbase.id] 2024-11-14T14:41:01,579 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:41:01,584 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-14T14:41:01,605 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 19ms. 2024-11-14T14:41:01,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46729-0x101a73a44750002, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:41:01,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46229-0x101a73a44750003, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:41:01,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34057-0x101a73a44750000, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:41:01,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40103-0x101a73a44750001, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:41:01,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45825 is added to blk_1073741827_1003 (size=196) 2024-11-14T14:41:01,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45021 is added to blk_1073741827_1003 (size=196) 2024-11-14T14:41:01,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36849 is added to blk_1073741827_1003 (size=196) 2024-11-14T14:41:01,648 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T14:41:01,650 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-14T14:41:01,669 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:41:01,674 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T14:41:01,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36849 is added to blk_1073741828_1004 (size=1189) 2024-11-14T14:41:01,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45021 is added to blk_1073741828_1004 (size=1189) 2024-11-14T14:41:01,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45825 is added to blk_1073741828_1004 (size=1189) 2024-11-14T14:41:01,740 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/MasterData/data/master/store 2024-11-14T14:41:01,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45825 is added to blk_1073741829_1005 (size=34) 2024-11-14T14:41:01,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36849 is added to blk_1073741829_1005 (size=34) 2024-11-14T14:41:01,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45021 is added to blk_1073741829_1005 (size=34) 2024-11-14T14:41:01,779 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-14T14:41:01,782 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:41:01,784 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-14T14:41:01,784 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:41:01,785 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:41:01,787 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-14T14:41:01,787 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:41:01,787 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:41:01,789 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731595261784Disabling compacts and flushes for region at 1731595261784Disabling writes for close at 1731595261787 (+3 ms)Writing region close event to WAL at 1731595261787Closed at 1731595261787 2024-11-14T14:41:01,792 WARN [master/da1c6afcd1f9:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/MasterData/data/master/store/.initializing 2024-11-14T14:41:01,792 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/MasterData/WALs/da1c6afcd1f9,34057,1731595260501 2024-11-14T14:41:01,805 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T14:41:01,827 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=da1c6afcd1f9%2C34057%2C1731595260501, suffix=, logDir=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/MasterData/WALs/da1c6afcd1f9,34057,1731595260501, archiveDir=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/MasterData/oldWALs, maxLogs=10 2024-11-14T14:41:01,862 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/MasterData/WALs/da1c6afcd1f9,34057,1731595260501/da1c6afcd1f9%2C34057%2C1731595260501.1731595261834, exclude list is [], retry=0 2024-11-14T14:41:01,884 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45021,DS-728ae2cb-a461-4860-83b4-16a012c6b9ad,DISK] 2024-11-14T14:41:01,884 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45825,DS-d2124e34-cef4-477f-b806-d1dbb55af1e5,DISK] 2024-11-14T14:41:01,884 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36849,DS-0f3bbfa7-353f-42f1-ac96-37bfae4542a6,DISK] 2024-11-14T14:41:01,887 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-14T14:41:01,943 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/MasterData/WALs/da1c6afcd1f9,34057,1731595260501/da1c6afcd1f9%2C34057%2C1731595260501.1731595261834 2024-11-14T14:41:01,944 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44937:44937),(127.0.0.1/127.0.0.1:34623:34623),(127.0.0.1/127.0.0.1:46259:46259)] 2024-11-14T14:41:01,944 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-14T14:41:01,945 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:41:01,948 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:41:01,950 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:41:02,007 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:41:02,042 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-14T14:41:02,047 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:41:02,050 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:41:02,051 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:41:02,054 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-14T14:41:02,054 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:41:02,055 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:41:02,056 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:41:02,058 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-14T14:41:02,058 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:41:02,059 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:41:02,059 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:41:02,062 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-14T14:41:02,062 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:41:02,062 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:41:02,063 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:41:02,066 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:41:02,067 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:41:02,072 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:41:02,073 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:41:02,076 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-14T14:41:02,079 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:41:02,084 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T14:41:02,085 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63074833, jitterRate=-0.060111746191978455}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-14T14:41:02,091 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731595261968Initializing all the Stores at 1731595261971 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731595261972 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595261973 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595261973Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595261973Cleaning up temporary data from old regions at 1731595262073 (+100 ms)Region opened successfully at 1731595262091 (+18 ms) 2024-11-14T14:41:02,092 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-14T14:41:02,127 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bd808e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=da1c6afcd1f9/172.17.0.2:0 2024-11-14T14:41:02,162 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-14T14:41:02,174 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-14T14:41:02,174 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-14T14:41:02,178 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T14:41:02,179 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-14T14:41:02,185 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-14T14:41:02,185 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-14T14:41:02,212 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-14T14:41:02,220 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34057-0x101a73a44750000, quorum=127.0.0.1:60650, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-14T14:41:02,223 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-14T14:41:02,225 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-14T14:41:02,227 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34057-0x101a73a44750000, quorum=127.0.0.1:60650, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-14T14:41:02,229 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-14T14:41:02,231 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-14T14:41:02,234 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34057-0x101a73a44750000, quorum=127.0.0.1:60650, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-14T14:41:02,236 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-14T14:41:02,237 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34057-0x101a73a44750000, quorum=127.0.0.1:60650, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-14T14:41:02,238 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-14T14:41:02,257 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34057-0x101a73a44750000, quorum=127.0.0.1:60650, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-14T14:41:02,260 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-14T14:41:02,265 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40103-0x101a73a44750001, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T14:41:02,265 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46229-0x101a73a44750003, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T14:41:02,265 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34057-0x101a73a44750000, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T14:41:02,265 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46729-0x101a73a44750002, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T14:41:02,265 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46229-0x101a73a44750003, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:41:02,265 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40103-0x101a73a44750001, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:41:02,266 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34057-0x101a73a44750000, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:41:02,269 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=da1c6afcd1f9,34057,1731595260501, sessionid=0x101a73a44750000, setting cluster-up flag (Was=false) 2024-11-14T14:41:02,272 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46729-0x101a73a44750002, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:41:02,285 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40103-0x101a73a44750001, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:41:02,285 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34057-0x101a73a44750000, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:41:02,285 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46229-0x101a73a44750003, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:41:02,285 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46729-0x101a73a44750002, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:41:02,293 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-14T14:41:02,296 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=da1c6afcd1f9,34057,1731595260501 2024-11-14T14:41:02,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34057-0x101a73a44750000, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:41:02,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46229-0x101a73a44750003, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:41:02,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40103-0x101a73a44750001, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:41:02,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46729-0x101a73a44750002, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:41:02,308 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-14T14:41:02,310 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=da1c6afcd1f9,34057,1731595260501 2024-11-14T14:41:02,317 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-14T14:41:02,393 INFO [RS:2;da1c6afcd1f9:46229 {}] regionserver.HRegionServer(746): ClusterId : cbb35874-c53a-4b8c-89da-bcc2cb2e4ea1 2024-11-14T14:41:02,393 INFO [RS:1;da1c6afcd1f9:46729 {}] regionserver.HRegionServer(746): ClusterId : cbb35874-c53a-4b8c-89da-bcc2cb2e4ea1 2024-11-14T14:41:02,395 INFO [RS:0;da1c6afcd1f9:40103 {}] regionserver.HRegionServer(746): ClusterId : cbb35874-c53a-4b8c-89da-bcc2cb2e4ea1 2024-11-14T14:41:02,396 DEBUG [RS:0;da1c6afcd1f9:40103 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-14T14:41:02,396 DEBUG [RS:2;da1c6afcd1f9:46229 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-14T14:41:02,396 DEBUG [RS:1;da1c6afcd1f9:46729 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-14T14:41:02,403 DEBUG [RS:0;da1c6afcd1f9:40103 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-14T14:41:02,403 DEBUG [RS:2;da1c6afcd1f9:46229 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-14T14:41:02,403 DEBUG [RS:1;da1c6afcd1f9:46729 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-14T14:41:02,403 DEBUG [RS:0;da1c6afcd1f9:40103 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-14T14:41:02,403 DEBUG [RS:2;da1c6afcd1f9:46229 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-14T14:41:02,403 DEBUG [RS:1;da1c6afcd1f9:46729 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-14T14:41:02,404 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T14:41:02,407 DEBUG [RS:1;da1c6afcd1f9:46729 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-14T14:41:02,407 DEBUG [RS:0;da1c6afcd1f9:40103 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-14T14:41:02,408 DEBUG [RS:1;da1c6afcd1f9:46729 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7010187d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=da1c6afcd1f9/172.17.0.2:0 2024-11-14T14:41:02,408 DEBUG [RS:0;da1c6afcd1f9:40103 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43b6c217, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=da1c6afcd1f9/172.17.0.2:0 2024-11-14T14:41:02,411 DEBUG [RS:2;da1c6afcd1f9:46229 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-14T14:41:02,412 DEBUG [RS:2;da1c6afcd1f9:46229 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c7ed0b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=da1c6afcd1f9/172.17.0.2:0 2024-11-14T14:41:02,419 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-14T14:41:02,428 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-14T14:41:02,432 DEBUG [RS:1;da1c6afcd1f9:46729 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;da1c6afcd1f9:46729 2024-11-14T14:41:02,437 DEBUG [RS:0;da1c6afcd1f9:40103 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;da1c6afcd1f9:40103 2024-11-14T14:41:02,437 INFO [RS:1;da1c6afcd1f9:46729 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-14T14:41:02,437 INFO [RS:0;da1c6afcd1f9:40103 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-14T14:41:02,437 INFO [RS:1;da1c6afcd1f9:46729 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-14T14:41:02,438 DEBUG [RS:1;da1c6afcd1f9:46729 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-14T14:41:02,438 INFO [RS:0;da1c6afcd1f9:40103 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-14T14:41:02,438 DEBUG [RS:2;da1c6afcd1f9:46229 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;da1c6afcd1f9:46229 2024-11-14T14:41:02,438 DEBUG [RS:0;da1c6afcd1f9:40103 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-14T14:41:02,438 INFO [RS:2;da1c6afcd1f9:46229 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-14T14:41:02,439 INFO [RS:2;da1c6afcd1f9:46229 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-14T14:41:02,439 DEBUG [RS:2;da1c6afcd1f9:46229 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-14T14:41:02,441 INFO [RS:1;da1c6afcd1f9:46729 {}] regionserver.HRegionServer(2659): reportForDuty to master=da1c6afcd1f9,34057,1731595260501 with port=46729, startcode=1731595261321 2024-11-14T14:41:02,441 INFO [RS:0;da1c6afcd1f9:40103 {}] regionserver.HRegionServer(2659): reportForDuty to master=da1c6afcd1f9,34057,1731595260501 with port=40103, startcode=1731595261206 2024-11-14T14:41:02,443 INFO [RS:2;da1c6afcd1f9:46229 {}] regionserver.HRegionServer(2659): reportForDuty to master=da1c6afcd1f9,34057,1731595260501 with port=46229, startcode=1731595261363 2024-11-14T14:41:02,440 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: da1c6afcd1f9,34057,1731595260501 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-14T14:41:02,450 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/da1c6afcd1f9:0, corePoolSize=5, maxPoolSize=5 2024-11-14T14:41:02,450 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/da1c6afcd1f9:0, corePoolSize=5, maxPoolSize=5 2024-11-14T14:41:02,450 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/da1c6afcd1f9:0, corePoolSize=5, maxPoolSize=5 2024-11-14T14:41:02,450 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/da1c6afcd1f9:0, corePoolSize=5, maxPoolSize=5 2024-11-14T14:41:02,450 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/da1c6afcd1f9:0, corePoolSize=10, maxPoolSize=10 2024-11-14T14:41:02,451 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:41:02,451 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/da1c6afcd1f9:0, corePoolSize=2, maxPoolSize=2 2024-11-14T14:41:02,451 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:41:02,458 DEBUG [RS:0;da1c6afcd1f9:40103 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-14T14:41:02,458 DEBUG [RS:2;da1c6afcd1f9:46229 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-14T14:41:02,458 DEBUG [RS:1;da1c6afcd1f9:46729 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-14T14:41:02,457 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731595292457 2024-11-14T14:41:02,460 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-14T14:41:02,462 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-14T14:41:02,466 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T14:41:02,466 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-14T14:41:02,467 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-14T14:41:02,467 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-14T14:41:02,467 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-14T14:41:02,468 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-14T14:41:02,474 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-14T14:41:02,477 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:41:02,477 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-14T14:41:02,479 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-14T14:41:02,480 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-14T14:41:02,481 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-14T14:41:02,484 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-14T14:41:02,485 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-14T14:41:02,490 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/da1c6afcd1f9:0:becomeActiveMaster-HFileCleaner.large.0-1731595262486,5,FailOnTimeoutGroup] 2024-11-14T14:41:02,500 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/da1c6afcd1f9:0:becomeActiveMaster-HFileCleaner.small.0-1731595262490,5,FailOnTimeoutGroup] 2024-11-14T14:41:02,500 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-14T14:41:02,500 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-14T14:41:02,504 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-14T14:41:02,506 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-14T14:41:02,507 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58159, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-14T14:41:02,507 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56649, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-14T14:41:02,508 INFO [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47633, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-14T14:41:02,538 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34057 {}] master.ServerManager(363): Checking decommissioned status of RegionServer da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:02,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36849 is added to blk_1073741831_1007 (size=1321) 2024-11-14T14:41:02,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45021 is added to blk_1073741831_1007 (size=1321) 2024-11-14T14:41:02,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45825 is added to blk_1073741831_1007 (size=1321) 2024-11-14T14:41:02,541 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34057 {}] master.ServerManager(517): Registering regionserver=da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:02,543 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-14T14:41:02,543 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b 2024-11-14T14:41:02,554 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34057 {}] master.ServerManager(363): Checking decommissioned status of RegionServer da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:02,555 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34057 {}] master.ServerManager(517): Registering regionserver=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:02,558 DEBUG [RS:0;da1c6afcd1f9:40103 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b 2024-11-14T14:41:02,559 DEBUG [RS:0;da1c6afcd1f9:40103 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:37365 2024-11-14T14:41:02,559 DEBUG [RS:0;da1c6afcd1f9:40103 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-14T14:41:02,560 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34057 {}] master.ServerManager(363): Checking decommissioned status of RegionServer da1c6afcd1f9,46729,1731595261321 2024-11-14T14:41:02,560 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34057 {}] master.ServerManager(517): Registering regionserver=da1c6afcd1f9,46729,1731595261321 2024-11-14T14:41:02,560 DEBUG [RS:2;da1c6afcd1f9:46229 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b 2024-11-14T14:41:02,560 DEBUG [RS:2;da1c6afcd1f9:46229 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:37365 2024-11-14T14:41:02,561 DEBUG [RS:2;da1c6afcd1f9:46229 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-14T14:41:02,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45825 is added to blk_1073741832_1008 (size=32) 2024-11-14T14:41:02,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45021 is added to blk_1073741832_1008 (size=32) 2024-11-14T14:41:02,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36849 is added to blk_1073741832_1008 (size=32) 2024-11-14T14:41:02,564 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34057-0x101a73a44750000, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T14:41:02,565 DEBUG [RS:1;da1c6afcd1f9:46729 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b 2024-11-14T14:41:02,565 DEBUG [RS:1;da1c6afcd1f9:46729 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:37365 2024-11-14T14:41:02,566 DEBUG [RS:1;da1c6afcd1f9:46729 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-14T14:41:02,568 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:41:02,568 DEBUG [RS:0;da1c6afcd1f9:40103 {}] zookeeper.ZKUtil(111): regionserver:40103-0x101a73a44750001, quorum=127.0.0.1:60650, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:02,568 WARN [RS:0;da1c6afcd1f9:40103 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T14:41:02,569 INFO [RS:0;da1c6afcd1f9:40103 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T14:41:02,569 DEBUG [RS:0;da1c6afcd1f9:40103 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/WALs/da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:02,569 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34057-0x101a73a44750000, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T14:41:02,569 DEBUG [RS:2;da1c6afcd1f9:46229 {}] zookeeper.ZKUtil(111): regionserver:46229-0x101a73a44750003, quorum=127.0.0.1:60650, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:02,570 WARN [RS:2;da1c6afcd1f9:46229 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T14:41:02,570 INFO [RS:2;da1c6afcd1f9:46229 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T14:41:02,570 DEBUG [RS:2;da1c6afcd1f9:46229 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/WALs/da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:02,570 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [da1c6afcd1f9,46229,1731595261363] 2024-11-14T14:41:02,570 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [da1c6afcd1f9,40103,1731595261206] 2024-11-14T14:41:02,572 DEBUG [RS:1;da1c6afcd1f9:46729 {}] zookeeper.ZKUtil(111): regionserver:46729-0x101a73a44750002, quorum=127.0.0.1:60650, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/da1c6afcd1f9,46729,1731595261321 2024-11-14T14:41:02,572 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [da1c6afcd1f9,46729,1731595261321] 2024-11-14T14:41:02,572 WARN [RS:1;da1c6afcd1f9:46729 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T14:41:02,572 INFO [RS:1;da1c6afcd1f9:46729 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T14:41:02,572 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-14T14:41:02,572 DEBUG [RS:1;da1c6afcd1f9:46729 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/WALs/da1c6afcd1f9,46729,1731595261321 2024-11-14T14:41:02,575 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T14:41:02,575 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:41:02,576 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:41:02,577 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-14T14:41:02,579 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T14:41:02,580 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:41:02,581 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:41:02,581 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-14T14:41:02,586 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T14:41:02,586 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:41:02,587 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:41:02,587 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-14T14:41:02,590 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T14:41:02,591 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:41:02,592 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:41:02,592 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T14:41:02,596 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/hbase/meta/1588230740 2024-11-14T14:41:02,597 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/hbase/meta/1588230740 2024-11-14T14:41:02,603 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T14:41:02,603 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T14:41:02,605 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-14T14:41:02,606 INFO [RS:0;da1c6afcd1f9:40103 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-14T14:41:02,606 INFO [RS:2;da1c6afcd1f9:46229 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-14T14:41:02,606 INFO [RS:1;da1c6afcd1f9:46729 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-14T14:41:02,607 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T14:41:02,612 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T14:41:02,613 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73933236, jitterRate=0.10169106721878052}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-14T14:41:02,617 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731595262568Initializing all the Stores at 1731595262570 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731595262570Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731595262571 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595262571Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731595262571Cleaning up temporary data from old regions at 1731595262604 (+33 ms)Region opened successfully at 1731595262617 (+13 ms) 2024-11-14T14:41:02,617 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T14:41:02,617 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T14:41:02,617 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T14:41:02,617 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T14:41:02,617 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T14:41:02,619 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T14:41:02,620 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731595262617Disabling compacts and flushes for region at 1731595262617Disabling writes for close at 1731595262617Writing region close event to WAL at 1731595262619 (+2 ms)Closed at 1731595262619 2024-11-14T14:41:02,623 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T14:41:02,623 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-14T14:41:02,627 INFO [RS:1;da1c6afcd1f9:46729 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-14T14:41:02,628 INFO [RS:2;da1c6afcd1f9:46229 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-14T14:41:02,628 INFO [RS:0;da1c6afcd1f9:40103 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-14T14:41:02,631 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-14T14:41:02,634 INFO [RS:1;da1c6afcd1f9:46729 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-14T14:41:02,634 INFO [RS:0;da1c6afcd1f9:40103 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-14T14:41:02,634 INFO [RS:2;da1c6afcd1f9:46229 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-14T14:41:02,634 INFO [RS:2;da1c6afcd1f9:46229 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T14:41:02,634 INFO [RS:1;da1c6afcd1f9:46729 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T14:41:02,634 INFO [RS:0;da1c6afcd1f9:40103 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T14:41:02,636 INFO [RS:1;da1c6afcd1f9:46729 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-14T14:41:02,636 INFO [RS:2;da1c6afcd1f9:46229 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-14T14:41:02,636 INFO [RS:0;da1c6afcd1f9:40103 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-14T14:41:02,644 INFO [RS:0;da1c6afcd1f9:40103 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-14T14:41:02,644 INFO [RS:2;da1c6afcd1f9:46229 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-14T14:41:02,644 INFO [RS:1;da1c6afcd1f9:46729 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-14T14:41:02,645 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T14:41:02,646 INFO [RS:2;da1c6afcd1f9:46229 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-14T14:41:02,646 INFO [RS:1;da1c6afcd1f9:46729 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-14T14:41:02,646 INFO [RS:0;da1c6afcd1f9:40103 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-14T14:41:02,646 DEBUG [RS:1;da1c6afcd1f9:46729 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:41:02,646 DEBUG [RS:2;da1c6afcd1f9:46229 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:41:02,647 DEBUG [RS:1;da1c6afcd1f9:46729 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:41:02,647 DEBUG [RS:2;da1c6afcd1f9:46229 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:41:02,647 DEBUG [RS:1;da1c6afcd1f9:46729 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:41:02,647 DEBUG [RS:2;da1c6afcd1f9:46229 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:41:02,647 DEBUG [RS:1;da1c6afcd1f9:46729 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:41:02,647 DEBUG [RS:2;da1c6afcd1f9:46229 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:41:02,647 DEBUG [RS:1;da1c6afcd1f9:46729 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:41:02,647 DEBUG [RS:2;da1c6afcd1f9:46229 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:41:02,647 DEBUG [RS:1;da1c6afcd1f9:46729 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/da1c6afcd1f9:0, corePoolSize=2, maxPoolSize=2 2024-11-14T14:41:02,647 DEBUG [RS:2;da1c6afcd1f9:46229 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/da1c6afcd1f9:0, corePoolSize=2, maxPoolSize=2 2024-11-14T14:41:02,646 DEBUG [RS:0;da1c6afcd1f9:40103 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:41:02,647 DEBUG [RS:1;da1c6afcd1f9:46729 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:41:02,648 DEBUG [RS:2;da1c6afcd1f9:46229 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:41:02,648 DEBUG [RS:0;da1c6afcd1f9:40103 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:41:02,648 DEBUG [RS:1;da1c6afcd1f9:46729 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:41:02,648 DEBUG [RS:2;da1c6afcd1f9:46229 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:41:02,648 DEBUG [RS:0;da1c6afcd1f9:40103 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:41:02,648 DEBUG [RS:1;da1c6afcd1f9:46729 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:41:02,648 DEBUG [RS:2;da1c6afcd1f9:46229 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:41:02,648 DEBUG [RS:0;da1c6afcd1f9:40103 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:41:02,648 DEBUG [RS:1;da1c6afcd1f9:46729 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:41:02,648 DEBUG [RS:2;da1c6afcd1f9:46229 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:41:02,648 DEBUG [RS:0;da1c6afcd1f9:40103 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:41:02,648 DEBUG [RS:2;da1c6afcd1f9:46229 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:41:02,648 DEBUG [RS:1;da1c6afcd1f9:46729 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:41:02,648 DEBUG [RS:0;da1c6afcd1f9:40103 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/da1c6afcd1f9:0, corePoolSize=2, maxPoolSize=2 2024-11-14T14:41:02,648 DEBUG [RS:2;da1c6afcd1f9:46229 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:41:02,648 DEBUG [RS:1;da1c6afcd1f9:46729 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:41:02,649 DEBUG [RS:0;da1c6afcd1f9:40103 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:41:02,649 DEBUG [RS:2;da1c6afcd1f9:46229 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/da1c6afcd1f9:0, corePoolSize=3, maxPoolSize=3 2024-11-14T14:41:02,649 DEBUG [RS:1;da1c6afcd1f9:46729 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/da1c6afcd1f9:0, corePoolSize=3, maxPoolSize=3 2024-11-14T14:41:02,649 DEBUG [RS:2;da1c6afcd1f9:46229 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/da1c6afcd1f9:0, corePoolSize=3, maxPoolSize=3 2024-11-14T14:41:02,649 DEBUG [RS:1;da1c6afcd1f9:46729 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/da1c6afcd1f9:0, corePoolSize=3, maxPoolSize=3 2024-11-14T14:41:02,649 DEBUG [RS:0;da1c6afcd1f9:40103 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:41:02,649 DEBUG [RS:0;da1c6afcd1f9:40103 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:41:02,649 DEBUG [RS:0;da1c6afcd1f9:40103 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:41:02,649 DEBUG [RS:0;da1c6afcd1f9:40103 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:41:02,650 DEBUG [RS:0;da1c6afcd1f9:40103 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:41:02,650 DEBUG [RS:0;da1c6afcd1f9:40103 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/da1c6afcd1f9:0, corePoolSize=3, maxPoolSize=3 2024-11-14T14:41:02,650 DEBUG [RS:0;da1c6afcd1f9:40103 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/da1c6afcd1f9:0, corePoolSize=3, maxPoolSize=3 2024-11-14T14:41:02,650 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-14T14:41:02,660 INFO [RS:1;da1c6afcd1f9:46729 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T14:41:02,660 INFO [RS:2;da1c6afcd1f9:46229 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T14:41:02,660 INFO [RS:1;da1c6afcd1f9:46729 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T14:41:02,660 INFO [RS:0;da1c6afcd1f9:40103 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T14:41:02,660 INFO [RS:2;da1c6afcd1f9:46229 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T14:41:02,660 INFO [RS:1;da1c6afcd1f9:46729 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T14:41:02,660 INFO [RS:2;da1c6afcd1f9:46229 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T14:41:02,660 INFO [RS:0;da1c6afcd1f9:40103 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T14:41:02,660 INFO [RS:1;da1c6afcd1f9:46729 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-14T14:41:02,660 INFO [RS:2;da1c6afcd1f9:46229 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-14T14:41:02,660 INFO [RS:0;da1c6afcd1f9:40103 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T14:41:02,660 INFO [RS:1;da1c6afcd1f9:46729 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-14T14:41:02,660 INFO [RS:2;da1c6afcd1f9:46229 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-14T14:41:02,660 INFO [RS:0;da1c6afcd1f9:40103 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-14T14:41:02,661 INFO [RS:1;da1c6afcd1f9:46729 {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,46729,1731595261321-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T14:41:02,661 INFO [RS:0;da1c6afcd1f9:40103 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-14T14:41:02,661 INFO [RS:2;da1c6afcd1f9:46229 {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,46229,1731595261363-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T14:41:02,661 INFO [RS:0;da1c6afcd1f9:40103 {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,40103,1731595261206-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T14:41:02,690 INFO [RS:0;da1c6afcd1f9:40103 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-14T14:41:02,690 INFO [RS:2;da1c6afcd1f9:46229 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-14T14:41:02,690 INFO [RS:1;da1c6afcd1f9:46729 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-14T14:41:02,693 INFO [RS:0;da1c6afcd1f9:40103 {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,40103,1731595261206-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T14:41:02,693 INFO [RS:1;da1c6afcd1f9:46729 {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,46729,1731595261321-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T14:41:02,693 INFO [RS:2;da1c6afcd1f9:46229 {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,46229,1731595261363-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T14:41:02,693 INFO [RS:0;da1c6afcd1f9:40103 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:41:02,693 INFO [RS:1;da1c6afcd1f9:46729 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:41:02,693 INFO [RS:2;da1c6afcd1f9:46229 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:41:02,694 INFO [RS:0;da1c6afcd1f9:40103 {}] regionserver.Replication(171): da1c6afcd1f9,40103,1731595261206 started 2024-11-14T14:41:02,694 INFO [RS:2;da1c6afcd1f9:46229 {}] regionserver.Replication(171): da1c6afcd1f9,46229,1731595261363 started 2024-11-14T14:41:02,694 INFO [RS:1;da1c6afcd1f9:46729 {}] regionserver.Replication(171): da1c6afcd1f9,46729,1731595261321 started 2024-11-14T14:41:02,715 INFO [RS:0;da1c6afcd1f9:40103 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:41:02,715 INFO [RS:0;da1c6afcd1f9:40103 {}] regionserver.HRegionServer(1482): Serving as da1c6afcd1f9,40103,1731595261206, RpcServer on da1c6afcd1f9/172.17.0.2:40103, sessionid=0x101a73a44750001 2024-11-14T14:41:02,716 DEBUG [RS:0;da1c6afcd1f9:40103 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-14T14:41:02,716 DEBUG [RS:0;da1c6afcd1f9:40103 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:02,716 DEBUG [RS:0;da1c6afcd1f9:40103 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'da1c6afcd1f9,40103,1731595261206' 2024-11-14T14:41:02,717 DEBUG [RS:0;da1c6afcd1f9:40103 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-14T14:41:02,718 DEBUG [RS:0;da1c6afcd1f9:40103 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-14T14:41:02,719 DEBUG [RS:0;da1c6afcd1f9:40103 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-14T14:41:02,719 DEBUG [RS:0;da1c6afcd1f9:40103 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-14T14:41:02,719 DEBUG [RS:0;da1c6afcd1f9:40103 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:02,719 DEBUG [RS:0;da1c6afcd1f9:40103 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'da1c6afcd1f9,40103,1731595261206' 2024-11-14T14:41:02,719 DEBUG [RS:0;da1c6afcd1f9:40103 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-14T14:41:02,719 INFO [RS:2;da1c6afcd1f9:46229 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:41:02,719 INFO [RS:1;da1c6afcd1f9:46729 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:41:02,719 INFO [RS:1;da1c6afcd1f9:46729 {}] regionserver.HRegionServer(1482): Serving as da1c6afcd1f9,46729,1731595261321, RpcServer on da1c6afcd1f9/172.17.0.2:46729, sessionid=0x101a73a44750002 2024-11-14T14:41:02,719 INFO [RS:2;da1c6afcd1f9:46229 {}] regionserver.HRegionServer(1482): Serving as da1c6afcd1f9,46229,1731595261363, RpcServer on da1c6afcd1f9/172.17.0.2:46229, sessionid=0x101a73a44750003 2024-11-14T14:41:02,719 DEBUG [RS:1;da1c6afcd1f9:46729 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-14T14:41:02,719 DEBUG [RS:2;da1c6afcd1f9:46229 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-14T14:41:02,720 DEBUG [RS:2;da1c6afcd1f9:46229 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:02,720 DEBUG [RS:1;da1c6afcd1f9:46729 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager da1c6afcd1f9,46729,1731595261321 2024-11-14T14:41:02,720 DEBUG [RS:1;da1c6afcd1f9:46729 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'da1c6afcd1f9,46729,1731595261321' 2024-11-14T14:41:02,720 DEBUG [RS:2;da1c6afcd1f9:46229 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'da1c6afcd1f9,46229,1731595261363' 2024-11-14T14:41:02,720 DEBUG [RS:2;da1c6afcd1f9:46229 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-14T14:41:02,720 DEBUG [RS:1;da1c6afcd1f9:46729 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-14T14:41:02,720 DEBUG [RS:0;da1c6afcd1f9:40103 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-14T14:41:02,721 DEBUG [RS:0;da1c6afcd1f9:40103 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-14T14:41:02,721 INFO [RS:0;da1c6afcd1f9:40103 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-14T14:41:02,721 INFO [RS:0;da1c6afcd1f9:40103 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-14T14:41:02,721 DEBUG [RS:2;da1c6afcd1f9:46229 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-14T14:41:02,721 DEBUG [RS:1;da1c6afcd1f9:46729 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-14T14:41:02,722 DEBUG [RS:2;da1c6afcd1f9:46229 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-14T14:41:02,722 DEBUG [RS:2;da1c6afcd1f9:46229 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-14T14:41:02,722 DEBUG [RS:2;da1c6afcd1f9:46229 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:02,722 DEBUG [RS:2;da1c6afcd1f9:46229 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'da1c6afcd1f9,46229,1731595261363' 2024-11-14T14:41:02,722 DEBUG [RS:1;da1c6afcd1f9:46729 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-14T14:41:02,722 DEBUG [RS:2;da1c6afcd1f9:46229 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-14T14:41:02,722 DEBUG [RS:1;da1c6afcd1f9:46729 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-14T14:41:02,722 DEBUG [RS:1;da1c6afcd1f9:46729 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager da1c6afcd1f9,46729,1731595261321 2024-11-14T14:41:02,722 DEBUG [RS:1;da1c6afcd1f9:46729 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'da1c6afcd1f9,46729,1731595261321' 2024-11-14T14:41:02,723 DEBUG [RS:1;da1c6afcd1f9:46729 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-14T14:41:02,723 DEBUG [RS:2;da1c6afcd1f9:46229 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-14T14:41:02,723 DEBUG [RS:1;da1c6afcd1f9:46729 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-14T14:41:02,724 DEBUG [RS:2;da1c6afcd1f9:46229 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-14T14:41:02,724 INFO [RS:2;da1c6afcd1f9:46229 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-14T14:41:02,724 INFO [RS:2;da1c6afcd1f9:46229 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-14T14:41:02,724 DEBUG [RS:1;da1c6afcd1f9:46729 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-14T14:41:02,724 INFO [RS:1;da1c6afcd1f9:46729 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-14T14:41:02,724 INFO [RS:1;da1c6afcd1f9:46729 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-14T14:41:02,801 WARN [da1c6afcd1f9:34057 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-14T14:41:02,827 INFO [RS:1;da1c6afcd1f9:46729 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T14:41:02,827 INFO [RS:2;da1c6afcd1f9:46229 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T14:41:02,827 INFO [RS:0;da1c6afcd1f9:40103 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T14:41:02,831 INFO [RS:0;da1c6afcd1f9:40103 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=da1c6afcd1f9%2C40103%2C1731595261206, suffix=, logDir=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/WALs/da1c6afcd1f9,40103,1731595261206, archiveDir=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/oldWALs, maxLogs=32 2024-11-14T14:41:02,831 INFO [RS:1;da1c6afcd1f9:46729 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=da1c6afcd1f9%2C46729%2C1731595261321, suffix=, logDir=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/WALs/da1c6afcd1f9,46729,1731595261321, archiveDir=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/oldWALs, maxLogs=32 2024-11-14T14:41:02,831 INFO [RS:2;da1c6afcd1f9:46229 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=da1c6afcd1f9%2C46229%2C1731595261363, suffix=, logDir=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/WALs/da1c6afcd1f9,46229,1731595261363, archiveDir=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/oldWALs, maxLogs=32 2024-11-14T14:41:02,850 DEBUG [RS:0;da1c6afcd1f9:40103 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/WALs/da1c6afcd1f9,40103,1731595261206/da1c6afcd1f9%2C40103%2C1731595261206.1731595262835, exclude list is [], retry=0 2024-11-14T14:41:02,851 DEBUG [RS:1;da1c6afcd1f9:46729 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/WALs/da1c6afcd1f9,46729,1731595261321/da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, exclude list is [], retry=0 2024-11-14T14:41:02,856 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45021,DS-728ae2cb-a461-4860-83b4-16a012c6b9ad,DISK] 2024-11-14T14:41:02,856 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36849,DS-0f3bbfa7-353f-42f1-ac96-37bfae4542a6,DISK] 2024-11-14T14:41:02,856 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45825,DS-d2124e34-cef4-477f-b806-d1dbb55af1e5,DISK] 2024-11-14T14:41:02,856 DEBUG [RS:2;da1c6afcd1f9:46229 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/WALs/da1c6afcd1f9,46229,1731595261363/da1c6afcd1f9%2C46229%2C1731595261363.1731595262836, exclude list is [], retry=0 2024-11-14T14:41:02,856 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45825,DS-d2124e34-cef4-477f-b806-d1dbb55af1e5,DISK] 2024-11-14T14:41:02,856 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36849,DS-0f3bbfa7-353f-42f1-ac96-37bfae4542a6,DISK] 2024-11-14T14:41:02,857 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45021,DS-728ae2cb-a461-4860-83b4-16a012c6b9ad,DISK] 2024-11-14T14:41:02,862 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36849,DS-0f3bbfa7-353f-42f1-ac96-37bfae4542a6,DISK] 2024-11-14T14:41:02,862 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45825,DS-d2124e34-cef4-477f-b806-d1dbb55af1e5,DISK] 2024-11-14T14:41:02,862 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45021,DS-728ae2cb-a461-4860-83b4-16a012c6b9ad,DISK] 2024-11-14T14:41:02,864 INFO [RS:0;da1c6afcd1f9:40103 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/WALs/da1c6afcd1f9,40103,1731595261206/da1c6afcd1f9%2C40103%2C1731595261206.1731595262835 2024-11-14T14:41:02,864 INFO [RS:1;da1c6afcd1f9:46729 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/WALs/da1c6afcd1f9,46729,1731595261321/da1c6afcd1f9%2C46729%2C1731595261321.1731595262836 2024-11-14T14:41:02,865 DEBUG [RS:0;da1c6afcd1f9:40103 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44937:44937),(127.0.0.1/127.0.0.1:46259:46259),(127.0.0.1/127.0.0.1:34623:34623)] 2024-11-14T14:41:02,865 DEBUG [RS:1;da1c6afcd1f9:46729 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46259:46259),(127.0.0.1/127.0.0.1:44937:44937),(127.0.0.1/127.0.0.1:34623:34623)] 2024-11-14T14:41:02,867 INFO [RS:2;da1c6afcd1f9:46229 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/WALs/da1c6afcd1f9,46229,1731595261363/da1c6afcd1f9%2C46229%2C1731595261363.1731595262836 2024-11-14T14:41:02,868 DEBUG [RS:2;da1c6afcd1f9:46229 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44937:44937),(127.0.0.1/127.0.0.1:34623:34623),(127.0.0.1/127.0.0.1:46259:46259)] 2024-11-14T14:41:03,054 DEBUG [da1c6afcd1f9:34057 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-14T14:41:03,063 DEBUG [da1c6afcd1f9:34057 {}] balancer.BalancerClusterState(204): Hosts are {da1c6afcd1f9=0} racks are {/default-rack=0} 2024-11-14T14:41:03,070 DEBUG [da1c6afcd1f9:34057 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-14T14:41:03,070 DEBUG [da1c6afcd1f9:34057 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-14T14:41:03,070 DEBUG [da1c6afcd1f9:34057 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-14T14:41:03,070 DEBUG [da1c6afcd1f9:34057 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-14T14:41:03,070 DEBUG [da1c6afcd1f9:34057 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-14T14:41:03,070 DEBUG [da1c6afcd1f9:34057 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-14T14:41:03,070 INFO [da1c6afcd1f9:34057 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-14T14:41:03,070 INFO [da1c6afcd1f9:34057 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-14T14:41:03,070 INFO [da1c6afcd1f9:34057 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-14T14:41:03,071 DEBUG [da1c6afcd1f9:34057 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-14T14:41:03,078 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=da1c6afcd1f9,46729,1731595261321 2024-11-14T14:41:03,085 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as da1c6afcd1f9,46729,1731595261321, state=OPENING 2024-11-14T14:41:03,091 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-14T14:41:03,093 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40103-0x101a73a44750001, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:41:03,093 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46229-0x101a73a44750003, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:41:03,093 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34057-0x101a73a44750000, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:41:03,093 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46729-0x101a73a44750002, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:41:03,094 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:03,094 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:03,094 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:03,094 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:03,095 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T14:41:03,097 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,46729,1731595261321}] 2024-11-14T14:41:03,273 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-14T14:41:03,275 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43793, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-14T14:41:03,287 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-14T14:41:03,288 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T14:41:03,288 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-14T14:41:03,291 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=da1c6afcd1f9%2C46729%2C1731595261321.meta, suffix=.meta, logDir=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/WALs/da1c6afcd1f9,46729,1731595261321, archiveDir=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/oldWALs, maxLogs=32 2024-11-14T14:41:03,307 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/WALs/da1c6afcd1f9,46729,1731595261321/da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, exclude list is [], retry=0 2024-11-14T14:41:03,311 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45021,DS-728ae2cb-a461-4860-83b4-16a012c6b9ad,DISK] 2024-11-14T14:41:03,311 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45825,DS-d2124e34-cef4-477f-b806-d1dbb55af1e5,DISK] 2024-11-14T14:41:03,312 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36849,DS-0f3bbfa7-353f-42f1-ac96-37bfae4542a6,DISK] 2024-11-14T14:41:03,315 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/WALs/da1c6afcd1f9,46729,1731595261321/da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta 2024-11-14T14:41:03,315 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34623:34623),(127.0.0.1/127.0.0.1:46259:46259),(127.0.0.1/127.0.0.1:44937:44937)] 2024-11-14T14:41:03,316 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-14T14:41:03,317 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-14T14:41:03,320 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-14T14:41:03,325 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-14T14:41:03,329 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-14T14:41:03,330 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:41:03,330 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-14T14:41:03,330 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-14T14:41:03,333 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-14T14:41:03,334 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T14:41:03,335 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:41:03,335 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:41:03,336 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-14T14:41:03,337 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T14:41:03,337 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:41:03,338 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:41:03,338 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-14T14:41:03,339 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T14:41:03,339 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:41:03,340 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:41:03,340 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-14T14:41:03,341 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T14:41:03,341 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:41:03,342 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:41:03,342 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T14:41:03,343 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/hbase/meta/1588230740 2024-11-14T14:41:03,346 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/hbase/meta/1588230740 2024-11-14T14:41:03,348 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T14:41:03,348 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T14:41:03,349 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-14T14:41:03,351 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T14:41:03,352 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64554993, jitterRate=-0.03805564343929291}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-14T14:41:03,353 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-14T14:41:03,354 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731595263330Writing region info on filesystem at 1731595263330Initializing all the Stores at 1731595263332 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731595263332Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731595263333 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595263333Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731595263333Cleaning up temporary data from old regions at 1731595263348 (+15 ms)Running coprocessor post-open hooks at 1731595263353 (+5 ms)Region opened successfully at 1731595263354 (+1 ms) 2024-11-14T14:41:03,362 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731595263264 2024-11-14T14:41:03,373 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-14T14:41:03,374 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-14T14:41:03,376 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=da1c6afcd1f9,46729,1731595261321 2024-11-14T14:41:03,378 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as da1c6afcd1f9,46729,1731595261321, state=OPEN 2024-11-14T14:41:03,380 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46229-0x101a73a44750003, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:03,381 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40103-0x101a73a44750001, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:03,381 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34057-0x101a73a44750000, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:03,381 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:03,381 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46729-0x101a73a44750002, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:03,381 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:03,381 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:03,381 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:03,381 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,46729,1731595261321 2024-11-14T14:41:03,387 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-14T14:41:03,387 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,46729,1731595261321 in 284 msec 2024-11-14T14:41:03,394 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-14T14:41:03,394 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 758 msec 2024-11-14T14:41:03,396 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T14:41:03,396 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-14T14:41:03,417 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T14:41:03,419 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=da1c6afcd1f9,46729,1731595261321, seqNum=-1] 2024-11-14T14:41:03,441 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T14:41:03,443 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36789, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T14:41:03,465 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1150 sec 2024-11-14T14:41:03,465 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731595263465, completionTime=-1 2024-11-14T14:41:03,468 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-14T14:41:03,468 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-14T14:41:03,496 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-14T14:41:03,496 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731595323496 2024-11-14T14:41:03,496 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731595383496 2024-11-14T14:41:03,496 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 28 msec 2024-11-14T14:41:03,498 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-14T14:41:03,506 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,34057,1731595260501-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T14:41:03,506 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,34057,1731595260501-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:41:03,506 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,34057,1731595260501-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:41:03,508 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-da1c6afcd1f9:34057, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:41:03,508 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-14T14:41:03,509 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-14T14:41:03,515 DEBUG [master/da1c6afcd1f9:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-14T14:41:03,537 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.090sec 2024-11-14T14:41:03,538 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-14T14:41:03,540 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-14T14:41:03,541 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-14T14:41:03,541 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-14T14:41:03,541 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-14T14:41:03,542 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,34057,1731595260501-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T14:41:03,542 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,34057,1731595260501-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-14T14:41:03,547 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-14T14:41:03,547 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-14T14:41:03,548 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,34057,1731595260501-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:41:03,604 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@385c0619, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T14:41:03,609 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-14T14:41:03,609 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-14T14:41:03,612 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request da1c6afcd1f9,34057,-1 for getting cluster id 2024-11-14T14:41:03,615 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T14:41:03,646 DEBUG [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbb35874-c53a-4b8c-89da-bcc2cb2e4ea1' 2024-11-14T14:41:03,650 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T14:41:03,650 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbb35874-c53a-4b8c-89da-bcc2cb2e4ea1" 2024-11-14T14:41:03,653 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4df07ecf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T14:41:03,653 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [da1c6afcd1f9,34057,-1] 2024-11-14T14:41:03,656 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T14:41:03,658 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:41:03,660 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46004, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T14:41:03,664 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27d156a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T14:41:03,664 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T14:41:03,676 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=da1c6afcd1f9,46729,1731595261321, seqNum=-1] 2024-11-14T14:41:03,677 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T14:41:03,679 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39240, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T14:41:03,702 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=da1c6afcd1f9,34057,1731595260501 2024-11-14T14:41:03,707 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T14:41:03,712 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is da1c6afcd1f9,34057,1731595260501 2024-11-14T14:41:03,715 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7031ec8b 2024-11-14T14:41:03,716 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T14:41:03,718 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46018, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T14:41:03,724 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34057 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'test', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T14:41:03,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34057 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test 2024-11-14T14:41:03,735 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T14:41:03,737 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34057 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "test" procId is: 4 2024-11-14T14:41:03,738 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:41:03,740 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T14:41:03,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T14:41:03,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45825 is added to blk_1073741837_1013 (size=358) 2024-11-14T14:41:03,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36849 is added to blk_1073741837_1013 (size=358) 2024-11-14T14:41:03,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45021 is added to blk_1073741837_1013 (size=358) 2024-11-14T14:41:03,769 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => c1c1eeeedb7ad89130474f95f0b19f70, NAME => 'test,ccc,1731595263719.c1c1eeeedb7ad89130474f95f0b19f70.', STARTKEY => 'ccc', ENDKEY => 'ddd'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b 2024-11-14T14:41:03,769 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => b7030563781ca9d5e4d044b06d29712c, NAME => 'test,bbb,1731595263719.b7030563781ca9d5e4d044b06d29712c.', STARTKEY => 'bbb', ENDKEY => 'ccc'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b 2024-11-14T14:41:03,769 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3613f06851ef4465281b91596c2f4aa4, NAME => 'test,,1731595263719.3613f06851ef4465281b91596c2f4aa4.', STARTKEY => '', ENDKEY => 'bbb'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b 2024-11-14T14:41:03,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36849 is added to blk_1073741838_1014 (size=45) 2024-11-14T14:41:03,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45021 is added to blk_1073741838_1014 (size=45) 2024-11-14T14:41:03,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45825 is added to blk_1073741838_1014 (size=45) 2024-11-14T14:41:03,813 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ccc,1731595263719.c1c1eeeedb7ad89130474f95f0b19f70.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:41:03,813 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing c1c1eeeedb7ad89130474f95f0b19f70, disabling compactions & flushes 2024-11-14T14:41:03,813 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ccc,1731595263719.c1c1eeeedb7ad89130474f95f0b19f70. 2024-11-14T14:41:03,813 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1731595263719.c1c1eeeedb7ad89130474f95f0b19f70. 2024-11-14T14:41:03,813 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1731595263719.c1c1eeeedb7ad89130474f95f0b19f70. after waiting 0 ms 2024-11-14T14:41:03,813 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1731595263719.c1c1eeeedb7ad89130474f95f0b19f70. 2024-11-14T14:41:03,813 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ccc,1731595263719.c1c1eeeedb7ad89130474f95f0b19f70. 2024-11-14T14:41:03,814 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for c1c1eeeedb7ad89130474f95f0b19f70: Waiting for close lock at 1731595263813Disabling compacts and flushes for region at 1731595263813Disabling writes for close at 1731595263813Writing region close event to WAL at 1731595263813Closed at 1731595263813 2024-11-14T14:41:03,815 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 8dbd58303faefb2f8b5972b0c3fc3aea, NAME => 'test,ddd,1731595263719.8dbd58303faefb2f8b5972b0c3fc3aea.', STARTKEY => 'ddd', ENDKEY => 'eee'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b 2024-11-14T14:41:03,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45825 is added to blk_1073741839_1015 (size=42) 2024-11-14T14:41:03,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45021 is added to blk_1073741839_1015 (size=42) 2024-11-14T14:41:03,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36849 is added to blk_1073741839_1015 (size=42) 2024-11-14T14:41:03,825 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,,1731595263719.3613f06851ef4465281b91596c2f4aa4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:41:03,825 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 3613f06851ef4465281b91596c2f4aa4, disabling compactions & flushes 2024-11-14T14:41:03,825 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,,1731595263719.3613f06851ef4465281b91596c2f4aa4. 2024-11-14T14:41:03,825 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1731595263719.3613f06851ef4465281b91596c2f4aa4. 2024-11-14T14:41:03,825 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,,1731595263719.3613f06851ef4465281b91596c2f4aa4. after waiting 0 ms 2024-11-14T14:41:03,825 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,,1731595263719.3613f06851ef4465281b91596c2f4aa4. 2024-11-14T14:41:03,825 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,,1731595263719.3613f06851ef4465281b91596c2f4aa4. 2024-11-14T14:41:03,825 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3613f06851ef4465281b91596c2f4aa4: Waiting for close lock at 1731595263825Disabling compacts and flushes for region at 1731595263825Disabling writes for close at 1731595263825Writing region close event to WAL at 1731595263825Closed at 1731595263825 2024-11-14T14:41:03,826 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5e71d8bcd86c479813fc8c2e3a01b917, NAME => 'test,eee,1731595263719.5e71d8bcd86c479813fc8c2e3a01b917.', STARTKEY => 'eee', ENDKEY => 'fff'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b 2024-11-14T14:41:03,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36849 is added to blk_1073741840_1016 (size=45) 2024-11-14T14:41:03,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45021 is added to blk_1073741840_1016 (size=45) 2024-11-14T14:41:03,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45825 is added to blk_1073741840_1016 (size=45) 2024-11-14T14:41:03,837 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,bbb,1731595263719.b7030563781ca9d5e4d044b06d29712c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:41:03,837 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing b7030563781ca9d5e4d044b06d29712c, disabling compactions & flushes 2024-11-14T14:41:03,837 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,bbb,1731595263719.b7030563781ca9d5e4d044b06d29712c. 2024-11-14T14:41:03,837 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1731595263719.b7030563781ca9d5e4d044b06d29712c. 2024-11-14T14:41:03,837 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1731595263719.b7030563781ca9d5e4d044b06d29712c. after waiting 0 ms 2024-11-14T14:41:03,837 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1731595263719.b7030563781ca9d5e4d044b06d29712c. 2024-11-14T14:41:03,837 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,bbb,1731595263719.b7030563781ca9d5e4d044b06d29712c. 2024-11-14T14:41:03,837 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for b7030563781ca9d5e4d044b06d29712c: Waiting for close lock at 1731595263837Disabling compacts and flushes for region at 1731595263837Disabling writes for close at 1731595263837Writing region close event to WAL at 1731595263837Closed at 1731595263837 2024-11-14T14:41:03,838 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => d35b9113ecfdcde0da6d3c51d6891527, NAME => 'test,fff,1731595263719.d35b9113ecfdcde0da6d3c51d6891527.', STARTKEY => 'fff', ENDKEY => 'ggg'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b 2024-11-14T14:41:03,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T14:41:03,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45021 is added to blk_1073741842_1018 (size=45) 2024-11-14T14:41:03,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36849 is added to blk_1073741841_1017 (size=45) 2024-11-14T14:41:03,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45825 is added to blk_1073741841_1017 (size=45) 2024-11-14T14:41:03,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45825 is added to blk_1073741842_1018 (size=45) 2024-11-14T14:41:03,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36849 is added to blk_1073741842_1018 (size=45) 2024-11-14T14:41:03,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45021 is added to blk_1073741841_1017 (size=45) 2024-11-14T14:41:03,858 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,eee,1731595263719.5e71d8bcd86c479813fc8c2e3a01b917.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:41:03,858 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ddd,1731595263719.8dbd58303faefb2f8b5972b0c3fc3aea.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:41:03,858 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 5e71d8bcd86c479813fc8c2e3a01b917, disabling compactions & flushes 2024-11-14T14:41:03,858 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,eee,1731595263719.5e71d8bcd86c479813fc8c2e3a01b917. 2024-11-14T14:41:03,858 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 8dbd58303faefb2f8b5972b0c3fc3aea, disabling compactions & flushes 2024-11-14T14:41:03,858 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1731595263719.5e71d8bcd86c479813fc8c2e3a01b917. 2024-11-14T14:41:03,858 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ddd,1731595263719.8dbd58303faefb2f8b5972b0c3fc3aea. 2024-11-14T14:41:03,858 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1731595263719.8dbd58303faefb2f8b5972b0c3fc3aea. 2024-11-14T14:41:03,858 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,eee,1731595263719.5e71d8bcd86c479813fc8c2e3a01b917. after waiting 0 ms 2024-11-14T14:41:03,858 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1731595263719.8dbd58303faefb2f8b5972b0c3fc3aea. after waiting 0 ms 2024-11-14T14:41:03,858 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,eee,1731595263719.5e71d8bcd86c479813fc8c2e3a01b917. 2024-11-14T14:41:03,858 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1731595263719.8dbd58303faefb2f8b5972b0c3fc3aea. 2024-11-14T14:41:03,858 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,eee,1731595263719.5e71d8bcd86c479813fc8c2e3a01b917. 2024-11-14T14:41:03,858 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ddd,1731595263719.8dbd58303faefb2f8b5972b0c3fc3aea. 2024-11-14T14:41:03,859 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5e71d8bcd86c479813fc8c2e3a01b917: Waiting for close lock at 1731595263858Disabling compacts and flushes for region at 1731595263858Disabling writes for close at 1731595263858Writing region close event to WAL at 1731595263858Closed at 1731595263858 2024-11-14T14:41:03,859 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 8dbd58303faefb2f8b5972b0c3fc3aea: Waiting for close lock at 1731595263858Disabling compacts and flushes for region at 1731595263858Disabling writes for close at 1731595263858Writing region close event to WAL at 1731595263858Closed at 1731595263858 2024-11-14T14:41:03,859 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9cb318690a770d1645dfd64bec1a284b, NAME => 'test,ggg,1731595263719.9cb318690a770d1645dfd64bec1a284b.', STARTKEY => 'ggg', ENDKEY => 'hhh'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b 2024-11-14T14:41:03,860 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => ad7ad94ee453836a534ae596990b42b8, NAME => 'test,hhh,1731595263719.ad7ad94ee453836a534ae596990b42b8.', STARTKEY => 'hhh', ENDKEY => 'iii'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b 2024-11-14T14:41:03,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36849 is added to blk_1073741843_1019 (size=45) 2024-11-14T14:41:03,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45825 is added to blk_1073741843_1019 (size=45) 2024-11-14T14:41:03,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45021 is added to blk_1073741843_1019 (size=45) 2024-11-14T14:41:03,869 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,fff,1731595263719.d35b9113ecfdcde0da6d3c51d6891527.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:41:03,870 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing d35b9113ecfdcde0da6d3c51d6891527, disabling compactions & flushes 2024-11-14T14:41:03,870 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,fff,1731595263719.d35b9113ecfdcde0da6d3c51d6891527. 2024-11-14T14:41:03,870 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1731595263719.d35b9113ecfdcde0da6d3c51d6891527. 2024-11-14T14:41:03,870 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,fff,1731595263719.d35b9113ecfdcde0da6d3c51d6891527. after waiting 0 ms 2024-11-14T14:41:03,870 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,fff,1731595263719.d35b9113ecfdcde0da6d3c51d6891527. 2024-11-14T14:41:03,870 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,fff,1731595263719.d35b9113ecfdcde0da6d3c51d6891527. 2024-11-14T14:41:03,870 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for d35b9113ecfdcde0da6d3c51d6891527: Waiting for close lock at 1731595263870Disabling compacts and flushes for region at 1731595263870Disabling writes for close at 1731595263870Writing region close event to WAL at 1731595263870Closed at 1731595263870 2024-11-14T14:41:03,870 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 135a286d1aafce40676bb4d4370c9daf, NAME => 'test,iii,1731595263719.135a286d1aafce40676bb4d4370c9daf.', STARTKEY => 'iii', ENDKEY => 'jjj'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b 2024-11-14T14:41:03,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45825 is added to blk_1073741845_1021 (size=45) 2024-11-14T14:41:03,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36849 is added to blk_1073741844_1020 (size=45) 2024-11-14T14:41:03,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45021 is added to blk_1073741845_1021 (size=45) 2024-11-14T14:41:03,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45825 is added to blk_1073741844_1020 (size=45) 2024-11-14T14:41:03,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36849 is added to blk_1073741845_1021 (size=45) 2024-11-14T14:41:03,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45021 is added to blk_1073741844_1020 (size=45) 2024-11-14T14:41:03,885 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ggg,1731595263719.9cb318690a770d1645dfd64bec1a284b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:41:03,885 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 9cb318690a770d1645dfd64bec1a284b, disabling compactions & flushes 2024-11-14T14:41:03,886 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ggg,1731595263719.9cb318690a770d1645dfd64bec1a284b. 2024-11-14T14:41:03,886 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1731595263719.9cb318690a770d1645dfd64bec1a284b. 2024-11-14T14:41:03,886 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1731595263719.9cb318690a770d1645dfd64bec1a284b. after waiting 0 ms 2024-11-14T14:41:03,886 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1731595263719.9cb318690a770d1645dfd64bec1a284b. 2024-11-14T14:41:03,886 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ggg,1731595263719.9cb318690a770d1645dfd64bec1a284b. 2024-11-14T14:41:03,886 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9cb318690a770d1645dfd64bec1a284b: Waiting for close lock at 1731595263885Disabling compacts and flushes for region at 1731595263885Disabling writes for close at 1731595263886 (+1 ms)Writing region close event to WAL at 1731595263886Closed at 1731595263886 2024-11-14T14:41:03,886 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,hhh,1731595263719.ad7ad94ee453836a534ae596990b42b8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:41:03,887 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing ad7ad94ee453836a534ae596990b42b8, disabling compactions & flushes 2024-11-14T14:41:03,887 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d14b6f516265d1b90f14cf4519cf2b0f, NAME => 'test,jjj,1731595263719.d14b6f516265d1b90f14cf4519cf2b0f.', STARTKEY => 'jjj', ENDKEY => 'kkk'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b 2024-11-14T14:41:03,887 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,hhh,1731595263719.ad7ad94ee453836a534ae596990b42b8. 2024-11-14T14:41:03,887 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1731595263719.ad7ad94ee453836a534ae596990b42b8. 2024-11-14T14:41:03,887 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1731595263719.ad7ad94ee453836a534ae596990b42b8. after waiting 0 ms 2024-11-14T14:41:03,887 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1731595263719.ad7ad94ee453836a534ae596990b42b8. 2024-11-14T14:41:03,887 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,hhh,1731595263719.ad7ad94ee453836a534ae596990b42b8. 2024-11-14T14:41:03,887 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for ad7ad94ee453836a534ae596990b42b8: Waiting for close lock at 1731595263887Disabling compacts and flushes for region at 1731595263887Disabling writes for close at 1731595263887Writing region close event to WAL at 1731595263887Closed at 1731595263887 2024-11-14T14:41:03,888 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 1d7e464b03c5060f7367637b64f2eae0, NAME => 'test,kkk,1731595263719.1d7e464b03c5060f7367637b64f2eae0.', STARTKEY => 'kkk', ENDKEY => 'lll'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b 2024-11-14T14:41:03,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36849 is added to blk_1073741846_1022 (size=45) 2024-11-14T14:41:03,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45021 is added to blk_1073741846_1022 (size=45) 2024-11-14T14:41:03,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45825 is added to blk_1073741846_1022 (size=45) 2024-11-14T14:41:03,895 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,iii,1731595263719.135a286d1aafce40676bb4d4370c9daf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:41:03,895 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 135a286d1aafce40676bb4d4370c9daf, disabling compactions & flushes 2024-11-14T14:41:03,895 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,iii,1731595263719.135a286d1aafce40676bb4d4370c9daf. 2024-11-14T14:41:03,895 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1731595263719.135a286d1aafce40676bb4d4370c9daf. 2024-11-14T14:41:03,895 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,iii,1731595263719.135a286d1aafce40676bb4d4370c9daf. after waiting 0 ms 2024-11-14T14:41:03,895 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,iii,1731595263719.135a286d1aafce40676bb4d4370c9daf. 2024-11-14T14:41:03,895 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,iii,1731595263719.135a286d1aafce40676bb4d4370c9daf. 2024-11-14T14:41:03,895 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 135a286d1aafce40676bb4d4370c9daf: Waiting for close lock at 1731595263895Disabling compacts and flushes for region at 1731595263895Disabling writes for close at 1731595263895Writing region close event to WAL at 1731595263895Closed at 1731595263895 2024-11-14T14:41:03,896 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 77f677c4c4cfcadb020d2436881cdeaa, NAME => 'test,lll,1731595263719.77f677c4c4cfcadb020d2436881cdeaa.', STARTKEY => 'lll', ENDKEY => 'mmm'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b 2024-11-14T14:41:03,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45021 is added to blk_1073741848_1024 (size=45) 2024-11-14T14:41:03,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45825 is added to blk_1073741847_1023 (size=45) 2024-11-14T14:41:03,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45021 is added to blk_1073741847_1023 (size=45) 2024-11-14T14:41:03,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45825 is added to blk_1073741848_1024 (size=45) 2024-11-14T14:41:03,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36849 is added to blk_1073741848_1024 (size=45) 2024-11-14T14:41:03,914 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,kkk,1731595263719.1d7e464b03c5060f7367637b64f2eae0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:41:03,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36849 is added to blk_1073741847_1023 (size=45) 2024-11-14T14:41:03,914 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 1d7e464b03c5060f7367637b64f2eae0, disabling compactions & flushes 2024-11-14T14:41:03,914 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,jjj,1731595263719.d14b6f516265d1b90f14cf4519cf2b0f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:41:03,914 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,kkk,1731595263719.1d7e464b03c5060f7367637b64f2eae0. 2024-11-14T14:41:03,914 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1731595263719.1d7e464b03c5060f7367637b64f2eae0. 2024-11-14T14:41:03,914 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing d14b6f516265d1b90f14cf4519cf2b0f, disabling compactions & flushes 2024-11-14T14:41:03,914 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1731595263719.1d7e464b03c5060f7367637b64f2eae0. after waiting 0 ms 2024-11-14T14:41:03,914 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1731595263719.1d7e464b03c5060f7367637b64f2eae0. 2024-11-14T14:41:03,914 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,jjj,1731595263719.d14b6f516265d1b90f14cf4519cf2b0f. 2024-11-14T14:41:03,914 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,kkk,1731595263719.1d7e464b03c5060f7367637b64f2eae0. 2024-11-14T14:41:03,914 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1731595263719.d14b6f516265d1b90f14cf4519cf2b0f. 2024-11-14T14:41:03,914 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 1d7e464b03c5060f7367637b64f2eae0: Waiting for close lock at 1731595263914Disabling compacts and flushes for region at 1731595263914Disabling writes for close at 1731595263914Writing region close event to WAL at 1731595263914Closed at 1731595263914 2024-11-14T14:41:03,914 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1731595263719.d14b6f516265d1b90f14cf4519cf2b0f. after waiting 0 ms 2024-11-14T14:41:03,914 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1731595263719.d14b6f516265d1b90f14cf4519cf2b0f. 2024-11-14T14:41:03,915 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,jjj,1731595263719.d14b6f516265d1b90f14cf4519cf2b0f. 2024-11-14T14:41:03,915 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 9f8e87df66c884919328ba09109d819c, NAME => 'test,mmm,1731595263719.9f8e87df66c884919328ba09109d819c.', STARTKEY => 'mmm', ENDKEY => 'nnn'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b 2024-11-14T14:41:03,915 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for d14b6f516265d1b90f14cf4519cf2b0f: Waiting for close lock at 1731595263914Disabling compacts and flushes for region at 1731595263914Disabling writes for close at 1731595263914Writing region close event to WAL at 1731595263915 (+1 ms)Closed at 1731595263915 2024-11-14T14:41:03,916 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f0bff83267f1352a4c903f322542cf14, NAME => 'test,nnn,1731595263719.f0bff83267f1352a4c903f322542cf14.', STARTKEY => 'nnn', ENDKEY => 'ooo'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b 2024-11-14T14:41:03,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36849 is added to blk_1073741849_1025 (size=45) 2024-11-14T14:41:03,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45825 is added to blk_1073741849_1025 (size=45) 2024-11-14T14:41:03,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45021 is added to blk_1073741849_1025 (size=45) 2024-11-14T14:41:03,925 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,lll,1731595263719.77f677c4c4cfcadb020d2436881cdeaa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:41:03,925 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 77f677c4c4cfcadb020d2436881cdeaa, disabling compactions & flushes 2024-11-14T14:41:03,925 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,lll,1731595263719.77f677c4c4cfcadb020d2436881cdeaa. 2024-11-14T14:41:03,925 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1731595263719.77f677c4c4cfcadb020d2436881cdeaa. 2024-11-14T14:41:03,925 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,lll,1731595263719.77f677c4c4cfcadb020d2436881cdeaa. after waiting 0 ms 2024-11-14T14:41:03,925 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,lll,1731595263719.77f677c4c4cfcadb020d2436881cdeaa. 2024-11-14T14:41:03,925 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,lll,1731595263719.77f677c4c4cfcadb020d2436881cdeaa. 2024-11-14T14:41:03,925 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 77f677c4c4cfcadb020d2436881cdeaa: Waiting for close lock at 1731595263925Disabling compacts and flushes for region at 1731595263925Disabling writes for close at 1731595263925Writing region close event to WAL at 1731595263925Closed at 1731595263925 2024-11-14T14:41:03,926 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 550a3973a29751310a867312ba0b03d1, NAME => 'test,ooo,1731595263719.550a3973a29751310a867312ba0b03d1.', STARTKEY => 'ooo', ENDKEY => 'ppp'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b 2024-11-14T14:41:03,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36849 is added to blk_1073741850_1026 (size=45) 2024-11-14T14:41:03,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45021 is added to blk_1073741850_1026 (size=45) 2024-11-14T14:41:03,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45825 is added to blk_1073741850_1026 (size=45) 2024-11-14T14:41:03,940 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,mmm,1731595263719.9f8e87df66c884919328ba09109d819c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:41:03,940 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 9f8e87df66c884919328ba09109d819c, disabling compactions & flushes 2024-11-14T14:41:03,940 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,mmm,1731595263719.9f8e87df66c884919328ba09109d819c. 2024-11-14T14:41:03,941 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1731595263719.9f8e87df66c884919328ba09109d819c. 2024-11-14T14:41:03,941 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1731595263719.9f8e87df66c884919328ba09109d819c. after waiting 0 ms 2024-11-14T14:41:03,941 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1731595263719.9f8e87df66c884919328ba09109d819c. 2024-11-14T14:41:03,941 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,mmm,1731595263719.9f8e87df66c884919328ba09109d819c. 2024-11-14T14:41:03,941 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 9f8e87df66c884919328ba09109d819c: Waiting for close lock at 1731595263940Disabling compacts and flushes for region at 1731595263940Disabling writes for close at 1731595263941 (+1 ms)Writing region close event to WAL at 1731595263941Closed at 1731595263941 2024-11-14T14:41:03,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45825 is added to blk_1073741851_1027 (size=45) 2024-11-14T14:41:03,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45021 is added to blk_1073741851_1027 (size=45) 2024-11-14T14:41:03,942 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 624639c373b3612dcb71af916020ab1e, NAME => 'test,ppp,1731595263719.624639c373b3612dcb71af916020ab1e.', STARTKEY => 'ppp', ENDKEY => 'qqq'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b 2024-11-14T14:41:03,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36849 is added to blk_1073741851_1027 (size=45) 2024-11-14T14:41:03,944 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,nnn,1731595263719.f0bff83267f1352a4c903f322542cf14.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:41:03,944 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing f0bff83267f1352a4c903f322542cf14, disabling compactions & flushes 2024-11-14T14:41:03,944 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,nnn,1731595263719.f0bff83267f1352a4c903f322542cf14. 2024-11-14T14:41:03,944 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1731595263719.f0bff83267f1352a4c903f322542cf14. 2024-11-14T14:41:03,944 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1731595263719.f0bff83267f1352a4c903f322542cf14. after waiting 0 ms 2024-11-14T14:41:03,944 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1731595263719.f0bff83267f1352a4c903f322542cf14. 2024-11-14T14:41:03,944 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,nnn,1731595263719.f0bff83267f1352a4c903f322542cf14. 2024-11-14T14:41:03,944 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for f0bff83267f1352a4c903f322542cf14: Waiting for close lock at 1731595263944Disabling compacts and flushes for region at 1731595263944Disabling writes for close at 1731595263944Writing region close event to WAL at 1731595263944Closed at 1731595263944 2024-11-14T14:41:03,945 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e1484a67e6b0eaf08cc5e1d6301f8319, NAME => 'test,qqq,1731595263719.e1484a67e6b0eaf08cc5e1d6301f8319.', STARTKEY => 'qqq', ENDKEY => 'rrr'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b 2024-11-14T14:41:03,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45825 is added to blk_1073741852_1028 (size=45) 2024-11-14T14:41:03,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45021 is added to blk_1073741852_1028 (size=45) 2024-11-14T14:41:03,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36849 is added to blk_1073741852_1028 (size=45) 2024-11-14T14:41:03,957 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ooo,1731595263719.550a3973a29751310a867312ba0b03d1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:41:03,957 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 550a3973a29751310a867312ba0b03d1, disabling compactions & flushes 2024-11-14T14:41:03,957 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ooo,1731595263719.550a3973a29751310a867312ba0b03d1. 2024-11-14T14:41:03,957 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1731595263719.550a3973a29751310a867312ba0b03d1. 2024-11-14T14:41:03,957 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1731595263719.550a3973a29751310a867312ba0b03d1. after waiting 0 ms 2024-11-14T14:41:03,957 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1731595263719.550a3973a29751310a867312ba0b03d1. 2024-11-14T14:41:03,957 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ooo,1731595263719.550a3973a29751310a867312ba0b03d1. 2024-11-14T14:41:03,957 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 550a3973a29751310a867312ba0b03d1: Waiting for close lock at 1731595263957Disabling compacts and flushes for region at 1731595263957Disabling writes for close at 1731595263957Writing region close event to WAL at 1731595263957Closed at 1731595263957 2024-11-14T14:41:03,958 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => b2f6569935d4c7faa1470e22f0b79a20, NAME => 'test,rrr,1731595263719.b2f6569935d4c7faa1470e22f0b79a20.', STARTKEY => 'rrr', ENDKEY => 'sss'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b 2024-11-14T14:41:03,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45021 is added to blk_1073741853_1029 (size=45) 2024-11-14T14:41:03,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45825 is added to blk_1073741853_1029 (size=45) 2024-11-14T14:41:03,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36849 is added to blk_1073741853_1029 (size=45) 2024-11-14T14:41:03,968 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ppp,1731595263719.624639c373b3612dcb71af916020ab1e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:41:03,968 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 624639c373b3612dcb71af916020ab1e, disabling compactions & flushes 2024-11-14T14:41:03,968 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ppp,1731595263719.624639c373b3612dcb71af916020ab1e. 2024-11-14T14:41:03,968 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1731595263719.624639c373b3612dcb71af916020ab1e. 2024-11-14T14:41:03,968 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1731595263719.624639c373b3612dcb71af916020ab1e. after waiting 0 ms 2024-11-14T14:41:03,968 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1731595263719.624639c373b3612dcb71af916020ab1e. 2024-11-14T14:41:03,968 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ppp,1731595263719.624639c373b3612dcb71af916020ab1e. 2024-11-14T14:41:03,968 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 624639c373b3612dcb71af916020ab1e: Waiting for close lock at 1731595263968Disabling compacts and flushes for region at 1731595263968Disabling writes for close at 1731595263968Writing region close event to WAL at 1731595263968Closed at 1731595263968 2024-11-14T14:41:03,969 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 177291d3bf2c8a0b1e9a3689929e9e50, NAME => 'test,sss,1731595263719.177291d3bf2c8a0b1e9a3689929e9e50.', STARTKEY => 'sss', ENDKEY => 'ttt'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b 2024-11-14T14:41:03,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45825 is added to blk_1073741854_1030 (size=45) 2024-11-14T14:41:03,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36849 is added to blk_1073741854_1030 (size=45) 2024-11-14T14:41:03,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45021 is added to blk_1073741854_1030 (size=45) 2024-11-14T14:41:03,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45021 is added to blk_1073741855_1031 (size=45) 2024-11-14T14:41:03,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36849 is added to blk_1073741855_1031 (size=45) 2024-11-14T14:41:03,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45825 is added to blk_1073741855_1031 (size=45) 2024-11-14T14:41:03,982 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,rrr,1731595263719.b2f6569935d4c7faa1470e22f0b79a20.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:41:03,982 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing b2f6569935d4c7faa1470e22f0b79a20, disabling compactions & flushes 2024-11-14T14:41:03,982 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,rrr,1731595263719.b2f6569935d4c7faa1470e22f0b79a20. 2024-11-14T14:41:03,982 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1731595263719.b2f6569935d4c7faa1470e22f0b79a20. 2024-11-14T14:41:03,982 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1731595263719.b2f6569935d4c7faa1470e22f0b79a20. after waiting 0 ms 2024-11-14T14:41:03,982 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1731595263719.b2f6569935d4c7faa1470e22f0b79a20. 2024-11-14T14:41:03,982 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,rrr,1731595263719.b2f6569935d4c7faa1470e22f0b79a20. 2024-11-14T14:41:03,982 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for b2f6569935d4c7faa1470e22f0b79a20: Waiting for close lock at 1731595263982Disabling compacts and flushes for region at 1731595263982Disabling writes for close at 1731595263982Writing region close event to WAL at 1731595263982Closed at 1731595263982 2024-11-14T14:41:03,983 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 71350ab931ae29ff2210bd9d6c3eeac5, NAME => 'test,ttt,1731595263719.71350ab931ae29ff2210bd9d6c3eeac5.', STARTKEY => 'ttt', ENDKEY => 'uuu'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b 2024-11-14T14:41:03,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45021 is added to blk_1073741856_1032 (size=45) 2024-11-14T14:41:03,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45825 is added to blk_1073741856_1032 (size=45) 2024-11-14T14:41:03,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36849 is added to blk_1073741856_1032 (size=45) 2024-11-14T14:41:03,989 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,sss,1731595263719.177291d3bf2c8a0b1e9a3689929e9e50.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:41:03,989 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 177291d3bf2c8a0b1e9a3689929e9e50, disabling compactions & flushes 2024-11-14T14:41:03,989 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,sss,1731595263719.177291d3bf2c8a0b1e9a3689929e9e50. 2024-11-14T14:41:03,989 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1731595263719.177291d3bf2c8a0b1e9a3689929e9e50. 2024-11-14T14:41:03,989 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,sss,1731595263719.177291d3bf2c8a0b1e9a3689929e9e50. after waiting 0 ms 2024-11-14T14:41:03,989 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,sss,1731595263719.177291d3bf2c8a0b1e9a3689929e9e50. 2024-11-14T14:41:03,989 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,sss,1731595263719.177291d3bf2c8a0b1e9a3689929e9e50. 2024-11-14T14:41:03,989 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 177291d3bf2c8a0b1e9a3689929e9e50: Waiting for close lock at 1731595263989Disabling compacts and flushes for region at 1731595263989Disabling writes for close at 1731595263989Writing region close event to WAL at 1731595263989Closed at 1731595263989 2024-11-14T14:41:03,990 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 6d725ecb42ad6f19c89d6ddaa229edc8, NAME => 'test,uuu,1731595263719.6d725ecb42ad6f19c89d6ddaa229edc8.', STARTKEY => 'uuu', ENDKEY => 'vvv'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b 2024-11-14T14:41:03,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45825 is added to blk_1073741857_1033 (size=45) 2024-11-14T14:41:03,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36849 is added to blk_1073741857_1033 (size=45) 2024-11-14T14:41:03,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45021 is added to blk_1073741857_1033 (size=45) 2024-11-14T14:41:03,995 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ttt,1731595263719.71350ab931ae29ff2210bd9d6c3eeac5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:41:03,995 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 71350ab931ae29ff2210bd9d6c3eeac5, disabling compactions & flushes 2024-11-14T14:41:03,995 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ttt,1731595263719.71350ab931ae29ff2210bd9d6c3eeac5. 2024-11-14T14:41:03,995 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1731595263719.71350ab931ae29ff2210bd9d6c3eeac5. 2024-11-14T14:41:03,995 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1731595263719.71350ab931ae29ff2210bd9d6c3eeac5. after waiting 0 ms 2024-11-14T14:41:03,995 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1731595263719.71350ab931ae29ff2210bd9d6c3eeac5. 2024-11-14T14:41:03,995 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ttt,1731595263719.71350ab931ae29ff2210bd9d6c3eeac5. 2024-11-14T14:41:03,996 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 71350ab931ae29ff2210bd9d6c3eeac5: Waiting for close lock at 1731595263995Disabling compacts and flushes for region at 1731595263995Disabling writes for close at 1731595263995Writing region close event to WAL at 1731595263995Closed at 1731595263995 2024-11-14T14:41:03,996 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 2e2ab5762f8d8f31683b4f4994048f3e, NAME => 'test,vvv,1731595263719.2e2ab5762f8d8f31683b4f4994048f3e.', STARTKEY => 'vvv', ENDKEY => 'www'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b 2024-11-14T14:41:04,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45021 is added to blk_1073741858_1034 (size=45) 2024-11-14T14:41:04,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45825 is added to blk_1073741858_1034 (size=45) 2024-11-14T14:41:04,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36849 is added to blk_1073741858_1034 (size=45) 2024-11-14T14:41:04,009 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,uuu,1731595263719.6d725ecb42ad6f19c89d6ddaa229edc8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:41:04,009 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 6d725ecb42ad6f19c89d6ddaa229edc8, disabling compactions & flushes 2024-11-14T14:41:04,009 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,uuu,1731595263719.6d725ecb42ad6f19c89d6ddaa229edc8. 2024-11-14T14:41:04,009 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1731595263719.6d725ecb42ad6f19c89d6ddaa229edc8. 2024-11-14T14:41:04,009 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1731595263719.6d725ecb42ad6f19c89d6ddaa229edc8. after waiting 0 ms 2024-11-14T14:41:04,009 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1731595263719.6d725ecb42ad6f19c89d6ddaa229edc8. 2024-11-14T14:41:04,009 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,uuu,1731595263719.6d725ecb42ad6f19c89d6ddaa229edc8. 2024-11-14T14:41:04,009 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 6d725ecb42ad6f19c89d6ddaa229edc8: Waiting for close lock at 1731595264009Disabling compacts and flushes for region at 1731595264009Disabling writes for close at 1731595264009Writing region close event to WAL at 1731595264009Closed at 1731595264009 2024-11-14T14:41:04,010 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => dbc3f9fabaa1bd9fca0801fe6494782e, NAME => 'test,www,1731595263719.dbc3f9fabaa1bd9fca0801fe6494782e.', STARTKEY => 'www', ENDKEY => 'xxx'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b 2024-11-14T14:41:04,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45021 is added to blk_1073741859_1035 (size=45) 2024-11-14T14:41:04,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45825 is added to blk_1073741859_1035 (size=45) 2024-11-14T14:41:04,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36849 is added to blk_1073741859_1035 (size=45) 2024-11-14T14:41:04,019 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,vvv,1731595263719.2e2ab5762f8d8f31683b4f4994048f3e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:41:04,019 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 2e2ab5762f8d8f31683b4f4994048f3e, disabling compactions & flushes 2024-11-14T14:41:04,019 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,vvv,1731595263719.2e2ab5762f8d8f31683b4f4994048f3e. 2024-11-14T14:41:04,019 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1731595263719.2e2ab5762f8d8f31683b4f4994048f3e. 2024-11-14T14:41:04,019 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1731595263719.2e2ab5762f8d8f31683b4f4994048f3e. after waiting 0 ms 2024-11-14T14:41:04,019 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1731595263719.2e2ab5762f8d8f31683b4f4994048f3e. 2024-11-14T14:41:04,019 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,vvv,1731595263719.2e2ab5762f8d8f31683b4f4994048f3e. 2024-11-14T14:41:04,019 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 2e2ab5762f8d8f31683b4f4994048f3e: Waiting for close lock at 1731595264019Disabling compacts and flushes for region at 1731595264019Disabling writes for close at 1731595264019Writing region close event to WAL at 1731595264019Closed at 1731595264019 2024-11-14T14:41:04,020 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 954bd925c25bfdb6e075a7471bbb1c4d, NAME => 'test,xxx,1731595263719.954bd925c25bfdb6e075a7471bbb1c4d.', STARTKEY => 'xxx', ENDKEY => 'yyy'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b 2024-11-14T14:41:04,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36849 is added to blk_1073741860_1036 (size=45) 2024-11-14T14:41:04,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45021 is added to blk_1073741860_1036 (size=45) 2024-11-14T14:41:04,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45825 is added to blk_1073741860_1036 (size=45) 2024-11-14T14:41:04,024 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,www,1731595263719.dbc3f9fabaa1bd9fca0801fe6494782e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:41:04,024 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing dbc3f9fabaa1bd9fca0801fe6494782e, disabling compactions & flushes 2024-11-14T14:41:04,024 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,www,1731595263719.dbc3f9fabaa1bd9fca0801fe6494782e. 2024-11-14T14:41:04,024 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1731595263719.dbc3f9fabaa1bd9fca0801fe6494782e. 2024-11-14T14:41:04,024 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,www,1731595263719.dbc3f9fabaa1bd9fca0801fe6494782e. after waiting 0 ms 2024-11-14T14:41:04,024 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,www,1731595263719.dbc3f9fabaa1bd9fca0801fe6494782e. 2024-11-14T14:41:04,024 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,www,1731595263719.dbc3f9fabaa1bd9fca0801fe6494782e. 2024-11-14T14:41:04,024 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for dbc3f9fabaa1bd9fca0801fe6494782e: Waiting for close lock at 1731595264024Disabling compacts and flushes for region at 1731595264024Disabling writes for close at 1731595264024Writing region close event to WAL at 1731595264024Closed at 1731595264024 2024-11-14T14:41:04,025 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 76be481b709a73fdf12e38c88e45d041, NAME => 'test,yyy,1731595263719.76be481b709a73fdf12e38c88e45d041.', STARTKEY => 'yyy', ENDKEY => 'zzz'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b 2024-11-14T14:41:04,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45021 is added to blk_1073741861_1037 (size=45) 2024-11-14T14:41:04,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36849 is added to blk_1073741861_1037 (size=45) 2024-11-14T14:41:04,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45825 is added to blk_1073741861_1037 (size=45) 2024-11-14T14:41:04,034 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,xxx,1731595263719.954bd925c25bfdb6e075a7471bbb1c4d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:41:04,034 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 954bd925c25bfdb6e075a7471bbb1c4d, disabling compactions & flushes 2024-11-14T14:41:04,034 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,xxx,1731595263719.954bd925c25bfdb6e075a7471bbb1c4d. 2024-11-14T14:41:04,034 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1731595263719.954bd925c25bfdb6e075a7471bbb1c4d. 2024-11-14T14:41:04,034 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1731595263719.954bd925c25bfdb6e075a7471bbb1c4d. after waiting 0 ms 2024-11-14T14:41:04,034 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1731595263719.954bd925c25bfdb6e075a7471bbb1c4d. 2024-11-14T14:41:04,034 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,xxx,1731595263719.954bd925c25bfdb6e075a7471bbb1c4d. 2024-11-14T14:41:04,035 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 954bd925c25bfdb6e075a7471bbb1c4d: Waiting for close lock at 1731595264034Disabling compacts and flushes for region at 1731595264034Disabling writes for close at 1731595264034Writing region close event to WAL at 1731595264034Closed at 1731595264034 2024-11-14T14:41:04,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45825 is added to blk_1073741862_1038 (size=45) 2024-11-14T14:41:04,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36849 is added to blk_1073741862_1038 (size=45) 2024-11-14T14:41:04,035 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1316438481dbf11d64fb665f6cdd4eb6, NAME => 'test,zzz,1731595263719.1316438481dbf11d64fb665f6cdd4eb6.', STARTKEY => 'zzz', ENDKEY => ''}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b 2024-11-14T14:41:04,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45021 is added to blk_1073741862_1038 (size=45) 2024-11-14T14:41:04,037 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,yyy,1731595263719.76be481b709a73fdf12e38c88e45d041.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:41:04,037 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 76be481b709a73fdf12e38c88e45d041, disabling compactions & flushes 2024-11-14T14:41:04,037 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,yyy,1731595263719.76be481b709a73fdf12e38c88e45d041. 2024-11-14T14:41:04,037 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1731595263719.76be481b709a73fdf12e38c88e45d041. 2024-11-14T14:41:04,037 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1731595263719.76be481b709a73fdf12e38c88e45d041. after waiting 0 ms 2024-11-14T14:41:04,037 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1731595263719.76be481b709a73fdf12e38c88e45d041. 2024-11-14T14:41:04,037 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,yyy,1731595263719.76be481b709a73fdf12e38c88e45d041. 2024-11-14T14:41:04,037 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 76be481b709a73fdf12e38c88e45d041: Waiting for close lock at 1731595264037Disabling compacts and flushes for region at 1731595264037Disabling writes for close at 1731595264037Writing region close event to WAL at 1731595264037Closed at 1731595264037 2024-11-14T14:41:04,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45825 is added to blk_1073741863_1039 (size=42) 2024-11-14T14:41:04,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36849 is added to blk_1073741863_1039 (size=42) 2024-11-14T14:41:04,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45021 is added to blk_1073741863_1039 (size=42) 2024-11-14T14:41:04,047 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,zzz,1731595263719.1316438481dbf11d64fb665f6cdd4eb6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:41:04,047 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 1316438481dbf11d64fb665f6cdd4eb6, disabling compactions & flushes 2024-11-14T14:41:04,047 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,zzz,1731595263719.1316438481dbf11d64fb665f6cdd4eb6. 2024-11-14T14:41:04,047 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1731595263719.1316438481dbf11d64fb665f6cdd4eb6. 2024-11-14T14:41:04,047 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1731595263719.1316438481dbf11d64fb665f6cdd4eb6. after waiting 0 ms 2024-11-14T14:41:04,047 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1731595263719.1316438481dbf11d64fb665f6cdd4eb6. 2024-11-14T14:41:04,047 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,zzz,1731595263719.1316438481dbf11d64fb665f6cdd4eb6. 2024-11-14T14:41:04,047 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 1316438481dbf11d64fb665f6cdd4eb6: Waiting for close lock at 1731595264047Disabling compacts and flushes for region at 1731595264047Disabling writes for close at 1731595264047Writing region close event to WAL at 1731595264047Closed at 1731595264047 2024-11-14T14:41:04,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T14:41:04,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T14:41:04,371 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,qqq,1731595263719.e1484a67e6b0eaf08cc5e1d6301f8319.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:41:04,372 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing e1484a67e6b0eaf08cc5e1d6301f8319, disabling compactions & flushes 2024-11-14T14:41:04,372 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,qqq,1731595263719.e1484a67e6b0eaf08cc5e1d6301f8319. 2024-11-14T14:41:04,372 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1731595263719.e1484a67e6b0eaf08cc5e1d6301f8319. 2024-11-14T14:41:04,372 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1731595263719.e1484a67e6b0eaf08cc5e1d6301f8319. after waiting 0 ms 2024-11-14T14:41:04,372 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1731595263719.e1484a67e6b0eaf08cc5e1d6301f8319. 2024-11-14T14:41:04,372 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,qqq,1731595263719.e1484a67e6b0eaf08cc5e1d6301f8319. 2024-11-14T14:41:04,372 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for e1484a67e6b0eaf08cc5e1d6301f8319: Waiting for close lock at 1731595264372Disabling compacts and flushes for region at 1731595264372Disabling writes for close at 1731595264372Writing region close event to WAL at 1731595264372Closed at 1731595264372 2024-11-14T14:41:04,374 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T14:41:04,380 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ccc,1731595263719.c1c1eeeedb7ad89130474f95f0b19f70.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731595264374"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731595264374"}]},"ts":"1731595264374"} 2024-11-14T14:41:04,380 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,,1731595263719.3613f06851ef4465281b91596c2f4aa4.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1731595264374"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731595264374"}]},"ts":"1731595264374"} 2024-11-14T14:41:04,380 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,bbb,1731595263719.b7030563781ca9d5e4d044b06d29712c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731595264374"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731595264374"}]},"ts":"1731595264374"} 2024-11-14T14:41:04,380 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,eee,1731595263719.5e71d8bcd86c479813fc8c2e3a01b917.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731595264374"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731595264374"}]},"ts":"1731595264374"} 2024-11-14T14:41:04,380 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ddd,1731595263719.8dbd58303faefb2f8b5972b0c3fc3aea.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731595264374"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731595264374"}]},"ts":"1731595264374"} 2024-11-14T14:41:04,381 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,fff,1731595263719.d35b9113ecfdcde0da6d3c51d6891527.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731595264374"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731595264374"}]},"ts":"1731595264374"} 2024-11-14T14:41:04,381 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ggg,1731595263719.9cb318690a770d1645dfd64bec1a284b.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731595264374"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731595264374"}]},"ts":"1731595264374"} 2024-11-14T14:41:04,381 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,hhh,1731595263719.ad7ad94ee453836a534ae596990b42b8.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731595264374"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731595264374"}]},"ts":"1731595264374"} 2024-11-14T14:41:04,381 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,iii,1731595263719.135a286d1aafce40676bb4d4370c9daf.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731595264374"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731595264374"}]},"ts":"1731595264374"} 2024-11-14T14:41:04,381 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,kkk,1731595263719.1d7e464b03c5060f7367637b64f2eae0.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731595264374"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731595264374"}]},"ts":"1731595264374"} 2024-11-14T14:41:04,381 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,jjj,1731595263719.d14b6f516265d1b90f14cf4519cf2b0f.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731595264374"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731595264374"}]},"ts":"1731595264374"} 2024-11-14T14:41:04,381 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,lll,1731595263719.77f677c4c4cfcadb020d2436881cdeaa.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731595264374"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731595264374"}]},"ts":"1731595264374"} 2024-11-14T14:41:04,382 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,mmm,1731595263719.9f8e87df66c884919328ba09109d819c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731595264374"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731595264374"}]},"ts":"1731595264374"} 2024-11-14T14:41:04,382 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,nnn,1731595263719.f0bff83267f1352a4c903f322542cf14.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731595264374"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731595264374"}]},"ts":"1731595264374"} 2024-11-14T14:41:04,382 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ooo,1731595263719.550a3973a29751310a867312ba0b03d1.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731595264374"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731595264374"}]},"ts":"1731595264374"} 2024-11-14T14:41:04,382 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ppp,1731595263719.624639c373b3612dcb71af916020ab1e.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731595264374"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731595264374"}]},"ts":"1731595264374"} 2024-11-14T14:41:04,382 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,rrr,1731595263719.b2f6569935d4c7faa1470e22f0b79a20.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731595264374"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731595264374"}]},"ts":"1731595264374"} 2024-11-14T14:41:04,382 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,sss,1731595263719.177291d3bf2c8a0b1e9a3689929e9e50.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731595264374"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731595264374"}]},"ts":"1731595264374"} 2024-11-14T14:41:04,382 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ttt,1731595263719.71350ab931ae29ff2210bd9d6c3eeac5.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731595264374"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731595264374"}]},"ts":"1731595264374"} 2024-11-14T14:41:04,383 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,uuu,1731595263719.6d725ecb42ad6f19c89d6ddaa229edc8.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731595264374"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731595264374"}]},"ts":"1731595264374"} 2024-11-14T14:41:04,383 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,vvv,1731595263719.2e2ab5762f8d8f31683b4f4994048f3e.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731595264374"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731595264374"}]},"ts":"1731595264374"} 2024-11-14T14:41:04,383 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,www,1731595263719.dbc3f9fabaa1bd9fca0801fe6494782e.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731595264374"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731595264374"}]},"ts":"1731595264374"} 2024-11-14T14:41:04,383 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,xxx,1731595263719.954bd925c25bfdb6e075a7471bbb1c4d.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731595264374"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731595264374"}]},"ts":"1731595264374"} 2024-11-14T14:41:04,383 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,yyy,1731595263719.76be481b709a73fdf12e38c88e45d041.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731595264374"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731595264374"}]},"ts":"1731595264374"} 2024-11-14T14:41:04,383 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,zzz,1731595263719.1316438481dbf11d64fb665f6cdd4eb6.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1731595264374"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731595264374"}]},"ts":"1731595264374"} 2024-11-14T14:41:04,383 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,qqq,1731595263719.e1484a67e6b0eaf08cc5e1d6301f8319.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731595264374"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731595264374"}]},"ts":"1731595264374"} 2024-11-14T14:41:04,441 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 26 regions to meta. 2024-11-14T14:41:04,443 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T14:41:04,446 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731595264443"}]},"ts":"1731595264443"} 2024-11-14T14:41:04,451 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLING in hbase:meta 2024-11-14T14:41:04,452 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {da1c6afcd1f9=0} racks are {/default-rack=0} 2024-11-14T14:41:04,471 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-14T14:41:04,471 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-14T14:41:04,471 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-14T14:41:04,471 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-14T14:41:04,471 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-14T14:41:04,471 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-14T14:41:04,471 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-14T14:41:04,471 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-14T14:41:04,471 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-14T14:41:04,471 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-14T14:41:04,474 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3613f06851ef4465281b91596c2f4aa4, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c1c1eeeedb7ad89130474f95f0b19f70, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8dbd58303faefb2f8b5972b0c3fc3aea, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5e71d8bcd86c479813fc8c2e3a01b917, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9cb318690a770d1645dfd64bec1a284b, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ad7ad94ee453836a534ae596990b42b8, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=135a286d1aafce40676bb4d4370c9daf, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d14b6f516265d1b90f14cf4519cf2b0f, ASSIGN}, {pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1d7e464b03c5060f7367637b64f2eae0, ASSIGN}, {pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN}, {pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9f8e87df66c884919328ba09109d819c, ASSIGN}, {pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f0bff83267f1352a4c903f322542cf14, ASSIGN}, {pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=550a3973a29751310a867312ba0b03d1, ASSIGN}, {pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=624639c373b3612dcb71af916020ab1e, ASSIGN}, {pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e1484a67e6b0eaf08cc5e1d6301f8319, ASSIGN}, {pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b2f6569935d4c7faa1470e22f0b79a20, ASSIGN}, {pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=177291d3bf2c8a0b1e9a3689929e9e50, ASSIGN}, {pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=71350ab931ae29ff2210bd9d6c3eeac5, ASSIGN}, {pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6d725ecb42ad6f19c89d6ddaa229edc8, ASSIGN}, {pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2e2ab5762f8d8f31683b4f4994048f3e, ASSIGN}, {pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=dbc3f9fabaa1bd9fca0801fe6494782e, ASSIGN}, {pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=954bd925c25bfdb6e075a7471bbb1c4d, ASSIGN}, {pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=76be481b709a73fdf12e38c88e45d041, ASSIGN}, {pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1316438481dbf11d64fb665f6cdd4eb6, ASSIGN}] 2024-11-14T14:41:04,482 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1316438481dbf11d64fb665f6cdd4eb6, ASSIGN 2024-11-14T14:41:04,484 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=1316438481dbf11d64fb665f6cdd4eb6, ASSIGN; state=OFFLINE, location=da1c6afcd1f9,46229,1731595261363; forceNewPlan=false, retain=false 2024-11-14T14:41:04,486 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=76be481b709a73fdf12e38c88e45d041, ASSIGN 2024-11-14T14:41:04,488 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=76be481b709a73fdf12e38c88e45d041, ASSIGN; state=OFFLINE, location=da1c6afcd1f9,46729,1731595261321; forceNewPlan=false, retain=false 2024-11-14T14:41:04,489 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=954bd925c25bfdb6e075a7471bbb1c4d, ASSIGN 2024-11-14T14:41:04,491 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=954bd925c25bfdb6e075a7471bbb1c4d, ASSIGN; state=OFFLINE, location=da1c6afcd1f9,40103,1731595261206; forceNewPlan=false, retain=false 2024-11-14T14:41:04,493 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=dbc3f9fabaa1bd9fca0801fe6494782e, ASSIGN 2024-11-14T14:41:04,494 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=dbc3f9fabaa1bd9fca0801fe6494782e, ASSIGN; state=OFFLINE, location=da1c6afcd1f9,46229,1731595261363; forceNewPlan=false, retain=false 2024-11-14T14:41:04,496 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2e2ab5762f8d8f31683b4f4994048f3e, ASSIGN 2024-11-14T14:41:04,497 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=2e2ab5762f8d8f31683b4f4994048f3e, ASSIGN; state=OFFLINE, location=da1c6afcd1f9,46729,1731595261321; forceNewPlan=false, retain=false 2024-11-14T14:41:04,499 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6d725ecb42ad6f19c89d6ddaa229edc8, ASSIGN 2024-11-14T14:41:04,500 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=6d725ecb42ad6f19c89d6ddaa229edc8, ASSIGN; state=OFFLINE, location=da1c6afcd1f9,40103,1731595261206; forceNewPlan=false, retain=false 2024-11-14T14:41:04,502 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=71350ab931ae29ff2210bd9d6c3eeac5, ASSIGN 2024-11-14T14:41:04,503 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=71350ab931ae29ff2210bd9d6c3eeac5, ASSIGN; state=OFFLINE, location=da1c6afcd1f9,46229,1731595261363; forceNewPlan=false, retain=false 2024-11-14T14:41:04,505 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=177291d3bf2c8a0b1e9a3689929e9e50, ASSIGN 2024-11-14T14:41:04,506 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=177291d3bf2c8a0b1e9a3689929e9e50, ASSIGN; state=OFFLINE, location=da1c6afcd1f9,46729,1731595261321; forceNewPlan=false, retain=false 2024-11-14T14:41:04,507 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b2f6569935d4c7faa1470e22f0b79a20, ASSIGN 2024-11-14T14:41:04,509 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=b2f6569935d4c7faa1470e22f0b79a20, ASSIGN; state=OFFLINE, location=da1c6afcd1f9,40103,1731595261206; forceNewPlan=false, retain=false 2024-11-14T14:41:04,510 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e1484a67e6b0eaf08cc5e1d6301f8319, ASSIGN 2024-11-14T14:41:04,511 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=e1484a67e6b0eaf08cc5e1d6301f8319, ASSIGN; state=OFFLINE, location=da1c6afcd1f9,40103,1731595261206; forceNewPlan=false, retain=false 2024-11-14T14:41:04,513 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=624639c373b3612dcb71af916020ab1e, ASSIGN 2024-11-14T14:41:04,514 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=624639c373b3612dcb71af916020ab1e, ASSIGN; state=OFFLINE, location=da1c6afcd1f9,46229,1731595261363; forceNewPlan=false, retain=false 2024-11-14T14:41:04,516 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=550a3973a29751310a867312ba0b03d1, ASSIGN 2024-11-14T14:41:04,517 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=550a3973a29751310a867312ba0b03d1, ASSIGN; state=OFFLINE, location=da1c6afcd1f9,46729,1731595261321; forceNewPlan=false, retain=false 2024-11-14T14:41:04,518 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f0bff83267f1352a4c903f322542cf14, ASSIGN 2024-11-14T14:41:04,520 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f0bff83267f1352a4c903f322542cf14, ASSIGN; state=OFFLINE, location=da1c6afcd1f9,40103,1731595261206; forceNewPlan=false, retain=false 2024-11-14T14:41:04,521 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9f8e87df66c884919328ba09109d819c, ASSIGN 2024-11-14T14:41:04,523 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=9f8e87df66c884919328ba09109d819c, ASSIGN; state=OFFLINE, location=da1c6afcd1f9,46229,1731595261363; forceNewPlan=false, retain=false 2024-11-14T14:41:04,524 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN 2024-11-14T14:41:04,526 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN; state=OFFLINE, location=da1c6afcd1f9,46729,1731595261321; forceNewPlan=false, retain=false 2024-11-14T14:41:04,527 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1d7e464b03c5060f7367637b64f2eae0, ASSIGN 2024-11-14T14:41:04,529 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=1d7e464b03c5060f7367637b64f2eae0, ASSIGN; state=OFFLINE, location=da1c6afcd1f9,46229,1731595261363; forceNewPlan=false, retain=false 2024-11-14T14:41:04,530 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d14b6f516265d1b90f14cf4519cf2b0f, ASSIGN 2024-11-14T14:41:04,531 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d14b6f516265d1b90f14cf4519cf2b0f, ASSIGN; state=OFFLINE, location=da1c6afcd1f9,40103,1731595261206; forceNewPlan=false, retain=false 2024-11-14T14:41:04,533 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=135a286d1aafce40676bb4d4370c9daf, ASSIGN 2024-11-14T14:41:04,534 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=135a286d1aafce40676bb4d4370c9daf, ASSIGN; state=OFFLINE, location=da1c6afcd1f9,46729,1731595261321; forceNewPlan=false, retain=false 2024-11-14T14:41:04,535 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ad7ad94ee453836a534ae596990b42b8, ASSIGN 2024-11-14T14:41:04,537 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ad7ad94ee453836a534ae596990b42b8, ASSIGN; state=OFFLINE, location=da1c6afcd1f9,40103,1731595261206; forceNewPlan=false, retain=false 2024-11-14T14:41:04,540 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9cb318690a770d1645dfd64bec1a284b, ASSIGN 2024-11-14T14:41:04,541 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=9cb318690a770d1645dfd64bec1a284b, ASSIGN; state=OFFLINE, location=da1c6afcd1f9,46229,1731595261363; forceNewPlan=false, retain=false 2024-11-14T14:41:04,543 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN 2024-11-14T14:41:04,544 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN; state=OFFLINE, location=da1c6afcd1f9,46729,1731595261321; forceNewPlan=false, retain=false 2024-11-14T14:41:04,546 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5e71d8bcd86c479813fc8c2e3a01b917, ASSIGN 2024-11-14T14:41:04,547 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=5e71d8bcd86c479813fc8c2e3a01b917, ASSIGN; state=OFFLINE, location=da1c6afcd1f9,46229,1731595261363; forceNewPlan=false, retain=false 2024-11-14T14:41:04,548 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8dbd58303faefb2f8b5972b0c3fc3aea, ASSIGN 2024-11-14T14:41:04,550 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8dbd58303faefb2f8b5972b0c3fc3aea, ASSIGN; state=OFFLINE, location=da1c6afcd1f9,40103,1731595261206; forceNewPlan=false, retain=false 2024-11-14T14:41:04,551 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c1c1eeeedb7ad89130474f95f0b19f70, ASSIGN 2024-11-14T14:41:04,552 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c1c1eeeedb7ad89130474f95f0b19f70, ASSIGN; state=OFFLINE, location=da1c6afcd1f9,46229,1731595261363; forceNewPlan=false, retain=false 2024-11-14T14:41:04,553 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN 2024-11-14T14:41:04,554 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN; state=OFFLINE, location=da1c6afcd1f9,46729,1731595261321; forceNewPlan=false, retain=false 2024-11-14T14:41:04,556 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3613f06851ef4465281b91596c2f4aa4, ASSIGN 2024-11-14T14:41:04,557 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=3613f06851ef4465281b91596c2f4aa4, ASSIGN; state=OFFLINE, location=da1c6afcd1f9,40103,1731595261206; forceNewPlan=false, retain=false 2024-11-14T14:41:04,638 INFO [da1c6afcd1f9:34057 {}] balancer.BaseLoadBalancer(388): Reassigned 26 regions. 26 retained the pre-restart assignment. 2024-11-14T14:41:04,639 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=76be481b709a73fdf12e38c88e45d041, regionState=OPENING, regionLocation=da1c6afcd1f9,46729,1731595261321 2024-11-14T14:41:04,640 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=2e2ab5762f8d8f31683b4f4994048f3e, regionState=OPENING, regionLocation=da1c6afcd1f9,46729,1731595261321 2024-11-14T14:41:04,641 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=177291d3bf2c8a0b1e9a3689929e9e50, regionState=OPENING, regionLocation=da1c6afcd1f9,46729,1731595261321 2024-11-14T14:41:04,641 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=550a3973a29751310a867312ba0b03d1, regionState=OPENING, regionLocation=da1c6afcd1f9,46729,1731595261321 2024-11-14T14:41:04,642 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=77f677c4c4cfcadb020d2436881cdeaa, regionState=OPENING, regionLocation=da1c6afcd1f9,46729,1731595261321 2024-11-14T14:41:04,643 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=135a286d1aafce40676bb4d4370c9daf, regionState=OPENING, regionLocation=da1c6afcd1f9,46729,1731595261321 2024-11-14T14:41:04,644 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=d35b9113ecfdcde0da6d3c51d6891527, regionState=OPENING, regionLocation=da1c6afcd1f9,46729,1731595261321 2024-11-14T14:41:04,644 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=76be481b709a73fdf12e38c88e45d041, ASSIGN because future has completed 2024-11-14T14:41:04,644 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=b7030563781ca9d5e4d044b06d29712c, regionState=OPENING, regionLocation=da1c6afcd1f9,46729,1731595261321 2024-11-14T14:41:04,645 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=1316438481dbf11d64fb665f6cdd4eb6, regionState=OPENING, regionLocation=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:04,645 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 76be481b709a73fdf12e38c88e45d041, server=da1c6afcd1f9,46729,1731595261321}] 2024-11-14T14:41:04,647 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=2e2ab5762f8d8f31683b4f4994048f3e, ASSIGN because future has completed 2024-11-14T14:41:04,651 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=177291d3bf2c8a0b1e9a3689929e9e50, ASSIGN because future has completed 2024-11-14T14:41:04,654 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 177291d3bf2c8a0b1e9a3689929e9e50, server=da1c6afcd1f9,46729,1731595261321}] 2024-11-14T14:41:04,655 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=550a3973a29751310a867312ba0b03d1, ASSIGN because future has completed 2024-11-14T14:41:04,658 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN because future has completed 2024-11-14T14:41:04,661 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=135a286d1aafce40676bb4d4370c9daf, ASSIGN because future has completed 2024-11-14T14:41:04,662 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 135a286d1aafce40676bb4d4370c9daf, server=da1c6afcd1f9,46729,1731595261321}] 2024-11-14T14:41:04,662 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN because future has completed 2024-11-14T14:41:04,670 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN because future has completed 2024-11-14T14:41:04,672 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=1316438481dbf11d64fb665f6cdd4eb6, ASSIGN because future has completed 2024-11-14T14:41:04,674 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1316438481dbf11d64fb665f6cdd4eb6, server=da1c6afcd1f9,46229,1731595261363}] 2024-11-14T14:41:04,679 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7030563781ca9d5e4d044b06d29712c, server=da1c6afcd1f9,46729,1731595261321}] 2024-11-14T14:41:04,684 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure d35b9113ecfdcde0da6d3c51d6891527, server=da1c6afcd1f9,46729,1731595261321}] 2024-11-14T14:41:04,689 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 77f677c4c4cfcadb020d2436881cdeaa, server=da1c6afcd1f9,46729,1731595261321}] 2024-11-14T14:41:04,695 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 550a3973a29751310a867312ba0b03d1, server=da1c6afcd1f9,46729,1731595261321}] 2024-11-14T14:41:04,700 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,46729,1731595261321}] 2024-11-14T14:41:04,705 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=dbc3f9fabaa1bd9fca0801fe6494782e, regionState=OPENING, regionLocation=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:04,706 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=71350ab931ae29ff2210bd9d6c3eeac5, regionState=OPENING, regionLocation=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:04,707 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=624639c373b3612dcb71af916020ab1e, regionState=OPENING, regionLocation=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:04,708 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=9f8e87df66c884919328ba09109d819c, regionState=OPENING, regionLocation=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:04,709 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=1d7e464b03c5060f7367637b64f2eae0, regionState=OPENING, regionLocation=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:04,709 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=dbc3f9fabaa1bd9fca0801fe6494782e, ASSIGN because future has completed 2024-11-14T14:41:04,709 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=9cb318690a770d1645dfd64bec1a284b, regionState=OPENING, regionLocation=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:04,710 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure dbc3f9fabaa1bd9fca0801fe6494782e, server=da1c6afcd1f9,46229,1731595261363}] 2024-11-14T14:41:04,711 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=71350ab931ae29ff2210bd9d6c3eeac5, ASSIGN because future has completed 2024-11-14T14:41:04,713 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=624639c373b3612dcb71af916020ab1e, ASSIGN because future has completed 2024-11-14T14:41:04,715 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=9f8e87df66c884919328ba09109d819c, ASSIGN because future has completed 2024-11-14T14:41:04,716 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=1d7e464b03c5060f7367637b64f2eae0, ASSIGN because future has completed 2024-11-14T14:41:04,717 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1d7e464b03c5060f7367637b64f2eae0, server=da1c6afcd1f9,46229,1731595261363}] 2024-11-14T14:41:04,718 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=9cb318690a770d1645dfd64bec1a284b, ASSIGN because future has completed 2024-11-14T14:41:04,722 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9cb318690a770d1645dfd64bec1a284b, server=da1c6afcd1f9,46229,1731595261363}] 2024-11-14T14:41:04,727 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9f8e87df66c884919328ba09109d819c, server=da1c6afcd1f9,46229,1731595261363}] 2024-11-14T14:41:04,732 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 624639c373b3612dcb71af916020ab1e, server=da1c6afcd1f9,46229,1731595261363}] 2024-11-14T14:41:04,736 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 71350ab931ae29ff2210bd9d6c3eeac5, server=da1c6afcd1f9,46229,1731595261363}] 2024-11-14T14:41:04,742 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=5e71d8bcd86c479813fc8c2e3a01b917, regionState=OPENING, regionLocation=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:04,743 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=c1c1eeeedb7ad89130474f95f0b19f70, regionState=OPENING, regionLocation=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:04,744 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=954bd925c25bfdb6e075a7471bbb1c4d, regionState=OPENING, regionLocation=da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:04,744 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=6d725ecb42ad6f19c89d6ddaa229edc8, regionState=OPENING, regionLocation=da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:04,745 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=b2f6569935d4c7faa1470e22f0b79a20, regionState=OPENING, regionLocation=da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:04,746 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=e1484a67e6b0eaf08cc5e1d6301f8319, regionState=OPENING, regionLocation=da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:04,747 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=5e71d8bcd86c479813fc8c2e3a01b917, ASSIGN because future has completed 2024-11-14T14:41:04,747 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=f0bff83267f1352a4c903f322542cf14, regionState=OPENING, regionLocation=da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:04,747 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5e71d8bcd86c479813fc8c2e3a01b917, server=da1c6afcd1f9,46229,1731595261363}] 2024-11-14T14:41:04,749 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c1c1eeeedb7ad89130474f95f0b19f70, ASSIGN because future has completed 2024-11-14T14:41:04,750 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=954bd925c25bfdb6e075a7471bbb1c4d, ASSIGN because future has completed 2024-11-14T14:41:04,752 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=6d725ecb42ad6f19c89d6ddaa229edc8, ASSIGN because future has completed 2024-11-14T14:41:04,753 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=b2f6569935d4c7faa1470e22f0b79a20, ASSIGN because future has completed 2024-11-14T14:41:04,754 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6d725ecb42ad6f19c89d6ddaa229edc8, server=da1c6afcd1f9,40103,1731595261206}] 2024-11-14T14:41:04,755 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=e1484a67e6b0eaf08cc5e1d6301f8319, ASSIGN because future has completed 2024-11-14T14:41:04,756 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f0bff83267f1352a4c903f322542cf14, ASSIGN because future has completed 2024-11-14T14:41:04,759 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure f0bff83267f1352a4c903f322542cf14, server=da1c6afcd1f9,40103,1731595261206}] 2024-11-14T14:41:04,764 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure e1484a67e6b0eaf08cc5e1d6301f8319, server=da1c6afcd1f9,40103,1731595261206}] 2024-11-14T14:41:04,768 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure b2f6569935d4c7faa1470e22f0b79a20, server=da1c6afcd1f9,40103,1731595261206}] 2024-11-14T14:41:04,773 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 954bd925c25bfdb6e075a7471bbb1c4d, server=da1c6afcd1f9,40103,1731595261206}] 2024-11-14T14:41:04,779 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure c1c1eeeedb7ad89130474f95f0b19f70, server=da1c6afcd1f9,46229,1731595261363}] 2024-11-14T14:41:04,785 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=d14b6f516265d1b90f14cf4519cf2b0f, regionState=OPENING, regionLocation=da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:04,786 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=ad7ad94ee453836a534ae596990b42b8, regionState=OPENING, regionLocation=da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:04,787 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=8dbd58303faefb2f8b5972b0c3fc3aea, regionState=OPENING, regionLocation=da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:04,788 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=3613f06851ef4465281b91596c2f4aa4, regionState=OPENING, regionLocation=da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:04,789 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d14b6f516265d1b90f14cf4519cf2b0f, ASSIGN because future has completed 2024-11-14T14:41:04,789 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure d14b6f516265d1b90f14cf4519cf2b0f, server=da1c6afcd1f9,40103,1731595261206}] 2024-11-14T14:41:04,790 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ad7ad94ee453836a534ae596990b42b8, ASSIGN because future has completed 2024-11-14T14:41:04,792 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8dbd58303faefb2f8b5972b0c3fc3aea, ASSIGN because future has completed 2024-11-14T14:41:04,793 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=3613f06851ef4465281b91596c2f4aa4, ASSIGN because future has completed 2024-11-14T14:41:04,796 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3613f06851ef4465281b91596c2f4aa4, server=da1c6afcd1f9,40103,1731595261206}] 2024-11-14T14:41:04,802 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8dbd58303faefb2f8b5972b0c3fc3aea, server=da1c6afcd1f9,40103,1731595261206}] 2024-11-14T14:41:04,807 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open test,fff,1731595263719.d35b9113ecfdcde0da6d3c51d6891527. 2024-11-14T14:41:04,808 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => d35b9113ecfdcde0da6d3c51d6891527, NAME => 'test,fff,1731595263719.d35b9113ecfdcde0da6d3c51d6891527.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-14T14:41:04,808 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d35b9113ecfdcde0da6d3c51d6891527 2024-11-14T14:41:04,808 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated test,fff,1731595263719.d35b9113ecfdcde0da6d3c51d6891527.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:41:04,809 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure ad7ad94ee453836a534ae596990b42b8, server=da1c6afcd1f9,40103,1731595261206}] 2024-11-14T14:41:04,809 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for d35b9113ecfdcde0da6d3c51d6891527 2024-11-14T14:41:04,809 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for d35b9113ecfdcde0da6d3c51d6891527 2024-11-14T14:41:04,814 INFO [StoreOpener-d35b9113ecfdcde0da6d3c51d6891527-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d35b9113ecfdcde0da6d3c51d6891527 2024-11-14T14:41:04,817 INFO [StoreOpener-d35b9113ecfdcde0da6d3c51d6891527-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d35b9113ecfdcde0da6d3c51d6891527 columnFamilyName family 2024-11-14T14:41:04,817 DEBUG [StoreOpener-d35b9113ecfdcde0da6d3c51d6891527-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:41:04,818 INFO [StoreOpener-d35b9113ecfdcde0da6d3c51d6891527-1 {}] regionserver.HStore(327): Store=d35b9113ecfdcde0da6d3c51d6891527/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:41:04,818 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for d35b9113ecfdcde0da6d3c51d6891527 2024-11-14T14:41:04,819 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/d35b9113ecfdcde0da6d3c51d6891527 2024-11-14T14:41:04,820 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/d35b9113ecfdcde0da6d3c51d6891527 2024-11-14T14:41:04,820 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for d35b9113ecfdcde0da6d3c51d6891527 2024-11-14T14:41:04,821 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for d35b9113ecfdcde0da6d3c51d6891527 2024-11-14T14:41:04,823 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for d35b9113ecfdcde0da6d3c51d6891527 2024-11-14T14:41:04,827 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/d35b9113ecfdcde0da6d3c51d6891527/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T14:41:04,827 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened d35b9113ecfdcde0da6d3c51d6891527; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73796313, jitterRate=0.09965075552463531}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T14:41:04,828 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d35b9113ecfdcde0da6d3c51d6891527 2024-11-14T14:41:04,829 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-14T14:41:04,829 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for d35b9113ecfdcde0da6d3c51d6891527: Running coprocessor pre-open hook at 1731595264809Writing region info on filesystem at 1731595264809Initializing all the Stores at 1731595264813 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595264813Cleaning up temporary data from old regions at 1731595264821 (+8 ms)Running coprocessor post-open hooks at 1731595264828 (+7 ms)Region opened successfully at 1731595264829 (+1 ms) 2024-11-14T14:41:04,832 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1731595263719.d35b9113ecfdcde0da6d3c51d6891527., pid=36, masterSystemTime=1731595264800 2024-11-14T14:41:04,833 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33835, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-14T14:41:04,836 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1731595263719.d35b9113ecfdcde0da6d3c51d6891527. 2024-11-14T14:41:04,837 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened test,fff,1731595263719.d35b9113ecfdcde0da6d3c51d6891527. 2024-11-14T14:41:04,837 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open test,sss,1731595263719.177291d3bf2c8a0b1e9a3689929e9e50. 2024-11-14T14:41:04,837 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 177291d3bf2c8a0b1e9a3689929e9e50, NAME => 'test,sss,1731595263719.177291d3bf2c8a0b1e9a3689929e9e50.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-14T14:41:04,837 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 177291d3bf2c8a0b1e9a3689929e9e50 2024-11-14T14:41:04,838 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated test,sss,1731595263719.177291d3bf2c8a0b1e9a3689929e9e50.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:41:04,838 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 177291d3bf2c8a0b1e9a3689929e9e50 2024-11-14T14:41:04,838 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 177291d3bf2c8a0b1e9a3689929e9e50 2024-11-14T14:41:04,840 INFO [StoreOpener-177291d3bf2c8a0b1e9a3689929e9e50-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 177291d3bf2c8a0b1e9a3689929e9e50 2024-11-14T14:41:04,841 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=d35b9113ecfdcde0da6d3c51d6891527, regionState=OPEN, openSeqNum=2, regionLocation=da1c6afcd1f9,46729,1731595261321 2024-11-14T14:41:04,842 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(132): Open test,ppp,1731595263719.624639c373b3612dcb71af916020ab1e. 2024-11-14T14:41:04,842 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7752): Opening region: {ENCODED => 624639c373b3612dcb71af916020ab1e, NAME => 'test,ppp,1731595263719.624639c373b3612dcb71af916020ab1e.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-14T14:41:04,842 INFO [StoreOpener-177291d3bf2c8a0b1e9a3689929e9e50-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 177291d3bf2c8a0b1e9a3689929e9e50 columnFamilyName family 2024-11-14T14:41:04,843 DEBUG [StoreOpener-177291d3bf2c8a0b1e9a3689929e9e50-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:41:04,843 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 624639c373b3612dcb71af916020ab1e 2024-11-14T14:41:04,843 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(898): Instantiated test,ppp,1731595263719.624639c373b3612dcb71af916020ab1e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:41:04,843 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7794): checking encryption for 624639c373b3612dcb71af916020ab1e 2024-11-14T14:41:04,843 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7797): checking classloading for 624639c373b3612dcb71af916020ab1e 2024-11-14T14:41:04,846 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure d35b9113ecfdcde0da6d3c51d6891527, server=da1c6afcd1f9,46729,1731595261321 because future has completed 2024-11-14T14:41:04,848 INFO [StoreOpener-624639c373b3612dcb71af916020ab1e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 624639c373b3612dcb71af916020ab1e 2024-11-14T14:41:04,850 INFO [StoreOpener-624639c373b3612dcb71af916020ab1e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 624639c373b3612dcb71af916020ab1e columnFamilyName family 2024-11-14T14:41:04,850 DEBUG [StoreOpener-624639c373b3612dcb71af916020ab1e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:41:04,851 INFO [StoreOpener-624639c373b3612dcb71af916020ab1e-1 {}] regionserver.HStore(327): Store=624639c373b3612dcb71af916020ab1e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:41:04,851 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1038): replaying wal for 624639c373b3612dcb71af916020ab1e 2024-11-14T14:41:04,852 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/624639c373b3612dcb71af916020ab1e 2024-11-14T14:41:04,852 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=10 2024-11-14T14:41:04,852 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure d35b9113ecfdcde0da6d3c51d6891527, server=da1c6afcd1f9,46729,1731595261321 in 163 msec 2024-11-14T14:41:04,853 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/624639c373b3612dcb71af916020ab1e 2024-11-14T14:41:04,854 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1048): stopping wal replay for 624639c373b3612dcb71af916020ab1e 2024-11-14T14:41:04,854 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1060): Cleaning up temporary data for 624639c373b3612dcb71af916020ab1e 2024-11-14T14:41:04,855 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN in 380 msec 2024-11-14T14:41:04,856 INFO [StoreOpener-177291d3bf2c8a0b1e9a3689929e9e50-1 {}] regionserver.HStore(327): Store=177291d3bf2c8a0b1e9a3689929e9e50/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:41:04,856 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 177291d3bf2c8a0b1e9a3689929e9e50 2024-11-14T14:41:04,857 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1093): writing seq id for 624639c373b3612dcb71af916020ab1e 2024-11-14T14:41:04,858 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/177291d3bf2c8a0b1e9a3689929e9e50 2024-11-14T14:41:04,858 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/177291d3bf2c8a0b1e9a3689929e9e50 2024-11-14T14:41:04,860 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 177291d3bf2c8a0b1e9a3689929e9e50 2024-11-14T14:41:04,860 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 177291d3bf2c8a0b1e9a3689929e9e50 2024-11-14T14:41:04,860 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/624639c373b3612dcb71af916020ab1e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T14:41:04,861 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1114): Opened 624639c373b3612dcb71af916020ab1e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66313933, jitterRate=-0.011845394968986511}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T14:41:04,861 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 624639c373b3612dcb71af916020ab1e 2024-11-14T14:41:04,862 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1006): Region open journal for 624639c373b3612dcb71af916020ab1e: Running coprocessor pre-open hook at 1731595264845Writing region info on filesystem at 1731595264845Initializing all the Stores at 1731595264847 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595264847Cleaning up temporary data from old regions at 1731595264854 (+7 ms)Running coprocessor post-open hooks at 1731595264861 (+7 ms)Region opened successfully at 1731595264862 (+1 ms) 2024-11-14T14:41:04,863 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 177291d3bf2c8a0b1e9a3689929e9e50 2024-11-14T14:41:04,863 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1731595263719.624639c373b3612dcb71af916020ab1e., pid=44, masterSystemTime=1731595264828 2024-11-14T14:41:04,866 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ppp,1731595263719.624639c373b3612dcb71af916020ab1e. 2024-11-14T14:41:04,866 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(153): Opened test,ppp,1731595263719.624639c373b3612dcb71af916020ab1e. 2024-11-14T14:41:04,866 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open test,eee,1731595263719.5e71d8bcd86c479813fc8c2e3a01b917. 2024-11-14T14:41:04,866 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/177291d3bf2c8a0b1e9a3689929e9e50/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T14:41:04,867 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => 5e71d8bcd86c479813fc8c2e3a01b917, NAME => 'test,eee,1731595263719.5e71d8bcd86c479813fc8c2e3a01b917.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-14T14:41:04,867 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=624639c373b3612dcb71af916020ab1e, regionState=OPEN, openSeqNum=2, regionLocation=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:04,867 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 5e71d8bcd86c479813fc8c2e3a01b917 2024-11-14T14:41:04,867 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 177291d3bf2c8a0b1e9a3689929e9e50; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68292480, jitterRate=0.017637252807617188}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T14:41:04,867 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated test,eee,1731595263719.5e71d8bcd86c479813fc8c2e3a01b917.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:41:04,867 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 177291d3bf2c8a0b1e9a3689929e9e50 2024-11-14T14:41:04,867 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for 5e71d8bcd86c479813fc8c2e3a01b917 2024-11-14T14:41:04,868 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 177291d3bf2c8a0b1e9a3689929e9e50: Running coprocessor pre-open hook at 1731595264838Writing region info on filesystem at 1731595264838Initializing all the Stores at 1731595264840 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595264840Cleaning up temporary data from old regions at 1731595264860 (+20 ms)Running coprocessor post-open hooks at 1731595264867 (+7 ms)Region opened successfully at 1731595264867 2024-11-14T14:41:04,868 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for 5e71d8bcd86c479813fc8c2e3a01b917 2024-11-14T14:41:04,869 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1731595263719.177291d3bf2c8a0b1e9a3689929e9e50., pid=32, masterSystemTime=1731595264800 2024-11-14T14:41:04,871 INFO [StoreOpener-5e71d8bcd86c479813fc8c2e3a01b917-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 5e71d8bcd86c479813fc8c2e3a01b917 2024-11-14T14:41:04,871 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 624639c373b3612dcb71af916020ab1e, server=da1c6afcd1f9,46229,1731595261363 because future has completed 2024-11-14T14:41:04,872 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1731595263719.177291d3bf2c8a0b1e9a3689929e9e50. 2024-11-14T14:41:04,872 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened test,sss,1731595263719.177291d3bf2c8a0b1e9a3689929e9e50. 2024-11-14T14:41:04,872 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open test,ooo,1731595263719.550a3973a29751310a867312ba0b03d1. 2024-11-14T14:41:04,873 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => 550a3973a29751310a867312ba0b03d1, NAME => 'test,ooo,1731595263719.550a3973a29751310a867312ba0b03d1.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-11-14T14:41:04,873 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 550a3973a29751310a867312ba0b03d1 2024-11-14T14:41:04,873 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated test,ooo,1731595263719.550a3973a29751310a867312ba0b03d1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:41:04,873 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for 550a3973a29751310a867312ba0b03d1 2024-11-14T14:41:04,873 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for 550a3973a29751310a867312ba0b03d1 2024-11-14T14:41:04,874 INFO [StoreOpener-5e71d8bcd86c479813fc8c2e3a01b917-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5e71d8bcd86c479813fc8c2e3a01b917 columnFamilyName family 2024-11-14T14:41:04,874 DEBUG [StoreOpener-5e71d8bcd86c479813fc8c2e3a01b917-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:41:04,875 INFO [StoreOpener-5e71d8bcd86c479813fc8c2e3a01b917-1 {}] regionserver.HStore(327): Store=5e71d8bcd86c479813fc8c2e3a01b917/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:41:04,875 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for 5e71d8bcd86c479813fc8c2e3a01b917 2024-11-14T14:41:04,876 INFO [StoreOpener-550a3973a29751310a867312ba0b03d1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 550a3973a29751310a867312ba0b03d1 2024-11-14T14:41:04,877 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/5e71d8bcd86c479813fc8c2e3a01b917 2024-11-14T14:41:04,877 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/5e71d8bcd86c479813fc8c2e3a01b917 2024-11-14T14:41:04,877 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=20 2024-11-14T14:41:04,878 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 624639c373b3612dcb71af916020ab1e, server=da1c6afcd1f9,46229,1731595261363 in 141 msec 2024-11-14T14:41:04,878 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for 5e71d8bcd86c479813fc8c2e3a01b917 2024-11-14T14:41:04,878 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for 5e71d8bcd86c479813fc8c2e3a01b917 2024-11-14T14:41:04,879 INFO [StoreOpener-550a3973a29751310a867312ba0b03d1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 550a3973a29751310a867312ba0b03d1 columnFamilyName family 2024-11-14T14:41:04,879 DEBUG [StoreOpener-550a3973a29751310a867312ba0b03d1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:41:04,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T14:41:04,880 INFO [StoreOpener-550a3973a29751310a867312ba0b03d1-1 {}] regionserver.HStore(327): Store=550a3973a29751310a867312ba0b03d1/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:41:04,881 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for 550a3973a29751310a867312ba0b03d1 2024-11-14T14:41:04,881 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=624639c373b3612dcb71af916020ab1e, ASSIGN in 405 msec 2024-11-14T14:41:04,882 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for 5e71d8bcd86c479813fc8c2e3a01b917 2024-11-14T14:41:04,882 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/550a3973a29751310a867312ba0b03d1 2024-11-14T14:41:04,882 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=177291d3bf2c8a0b1e9a3689929e9e50, regionState=OPEN, openSeqNum=2, regionLocation=da1c6afcd1f9,46729,1731595261321 2024-11-14T14:41:04,883 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/550a3973a29751310a867312ba0b03d1 2024-11-14T14:41:04,883 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for 550a3973a29751310a867312ba0b03d1 2024-11-14T14:41:04,883 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for 550a3973a29751310a867312ba0b03d1 2024-11-14T14:41:04,886 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 177291d3bf2c8a0b1e9a3689929e9e50, server=da1c6afcd1f9,46729,1731595261321 because future has completed 2024-11-14T14:41:04,887 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for 550a3973a29751310a867312ba0b03d1 2024-11-14T14:41:04,892 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=23 2024-11-14T14:41:04,892 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 177291d3bf2c8a0b1e9a3689929e9e50, server=da1c6afcd1f9,46729,1731595261321 in 235 msec 2024-11-14T14:41:04,893 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/5e71d8bcd86c479813fc8c2e3a01b917/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T14:41:04,894 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened 5e71d8bcd86c479813fc8c2e3a01b917; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75178981, jitterRate=0.12025411427021027}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T14:41:04,894 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5e71d8bcd86c479813fc8c2e3a01b917 2024-11-14T14:41:04,894 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for 5e71d8bcd86c479813fc8c2e3a01b917: Running coprocessor pre-open hook at 1731595264868Writing region info on filesystem at 1731595264868Initializing all the Stores at 1731595264870 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595264870Cleaning up temporary data from old regions at 1731595264878 (+8 ms)Running coprocessor post-open hooks at 1731595264894 (+16 ms)Region opened successfully at 1731595264894 2024-11-14T14:41:04,894 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=177291d3bf2c8a0b1e9a3689929e9e50, ASSIGN in 419 msec 2024-11-14T14:41:04,895 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1731595263719.5e71d8bcd86c479813fc8c2e3a01b917., pid=46, masterSystemTime=1731595264828 2024-11-14T14:41:04,898 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1731595263719.5e71d8bcd86c479813fc8c2e3a01b917. 2024-11-14T14:41:04,898 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened test,eee,1731595263719.5e71d8bcd86c479813fc8c2e3a01b917. 2024-11-14T14:41:04,898 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open test,mmm,1731595263719.9f8e87df66c884919328ba09109d819c. 2024-11-14T14:41:04,898 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => 9f8e87df66c884919328ba09109d819c, NAME => 'test,mmm,1731595263719.9f8e87df66c884919328ba09109d819c.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-14T14:41:04,899 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=5e71d8bcd86c479813fc8c2e3a01b917, regionState=OPEN, openSeqNum=2, regionLocation=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:04,899 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 9f8e87df66c884919328ba09109d819c 2024-11-14T14:41:04,899 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated test,mmm,1731595263719.9f8e87df66c884919328ba09109d819c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:41:04,899 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for 9f8e87df66c884919328ba09109d819c 2024-11-14T14:41:04,899 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for 9f8e87df66c884919328ba09109d819c 2024-11-14T14:41:04,901 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/550a3973a29751310a867312ba0b03d1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T14:41:04,902 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5e71d8bcd86c479813fc8c2e3a01b917, server=da1c6afcd1f9,46229,1731595261363 because future has completed 2024-11-14T14:41:04,902 INFO [StoreOpener-9f8e87df66c884919328ba09109d819c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 9f8e87df66c884919328ba09109d819c 2024-11-14T14:41:04,902 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened 550a3973a29751310a867312ba0b03d1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68133523, jitterRate=0.015268608927726746}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T14:41:04,902 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 550a3973a29751310a867312ba0b03d1 2024-11-14T14:41:04,903 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for 550a3973a29751310a867312ba0b03d1: Running coprocessor pre-open hook at 1731595264873Writing region info on filesystem at 1731595264873Initializing all the Stores at 1731595264875 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595264875Cleaning up temporary data from old regions at 1731595264883 (+8 ms)Running coprocessor post-open hooks at 1731595264902 (+19 ms)Region opened successfully at 1731595264903 (+1 ms) 2024-11-14T14:41:04,904 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ooo,1731595263719.550a3973a29751310a867312ba0b03d1., pid=38, masterSystemTime=1731595264800 2024-11-14T14:41:04,904 INFO [StoreOpener-9f8e87df66c884919328ba09109d819c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9f8e87df66c884919328ba09109d819c columnFamilyName family 2024-11-14T14:41:04,904 DEBUG [StoreOpener-9f8e87df66c884919328ba09109d819c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:41:04,905 INFO [StoreOpener-9f8e87df66c884919328ba09109d819c-1 {}] regionserver.HStore(327): Store=9f8e87df66c884919328ba09109d819c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:41:04,905 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for 9f8e87df66c884919328ba09109d819c 2024-11-14T14:41:04,906 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-14T14:41:04,908 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ooo,1731595263719.550a3973a29751310a867312ba0b03d1. 2024-11-14T14:41:04,908 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened test,ooo,1731595263719.550a3973a29751310a867312ba0b03d1. 2024-11-14T14:41:04,908 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open test,iii,1731595263719.135a286d1aafce40676bb4d4370c9daf. 2024-11-14T14:41:04,908 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/9f8e87df66c884919328ba09109d819c 2024-11-14T14:41:04,908 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 135a286d1aafce40676bb4d4370c9daf, NAME => 'test,iii,1731595263719.135a286d1aafce40676bb4d4370c9daf.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-14T14:41:04,909 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47003, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-14T14:41:04,909 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=9 2024-11-14T14:41:04,909 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure 5e71d8bcd86c479813fc8c2e3a01b917, server=da1c6afcd1f9,46229,1731595261363 in 156 msec 2024-11-14T14:41:04,909 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 135a286d1aafce40676bb4d4370c9daf 2024-11-14T14:41:04,909 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated test,iii,1731595263719.135a286d1aafce40676bb4d4370c9daf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:41:04,909 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 135a286d1aafce40676bb4d4370c9daf 2024-11-14T14:41:04,909 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 135a286d1aafce40676bb4d4370c9daf 2024-11-14T14:41:04,910 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/9f8e87df66c884919328ba09109d819c 2024-11-14T14:41:04,911 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for 9f8e87df66c884919328ba09109d819c 2024-11-14T14:41:04,911 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for 9f8e87df66c884919328ba09109d819c 2024-11-14T14:41:04,914 INFO [StoreOpener-135a286d1aafce40676bb4d4370c9daf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 135a286d1aafce40676bb4d4370c9daf 2024-11-14T14:41:04,914 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for 9f8e87df66c884919328ba09109d819c 2024-11-14T14:41:04,916 INFO [StoreOpener-135a286d1aafce40676bb4d4370c9daf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 135a286d1aafce40676bb4d4370c9daf columnFamilyName family 2024-11-14T14:41:04,916 DEBUG [StoreOpener-135a286d1aafce40676bb4d4370c9daf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:41:04,916 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=5e71d8bcd86c479813fc8c2e3a01b917, ASSIGN in 436 msec 2024-11-14T14:41:04,918 INFO [StoreOpener-135a286d1aafce40676bb4d4370c9daf-1 {}] regionserver.HStore(327): Store=135a286d1aafce40676bb4d4370c9daf/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:41:04,918 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open test,jjj,1731595263719.d14b6f516265d1b90f14cf4519cf2b0f. 2024-11-14T14:41:04,918 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=550a3973a29751310a867312ba0b03d1, regionState=OPEN, openSeqNum=2, regionLocation=da1c6afcd1f9,46729,1731595261321 2024-11-14T14:41:04,918 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => d14b6f516265d1b90f14cf4519cf2b0f, NAME => 'test,jjj,1731595263719.d14b6f516265d1b90f14cf4519cf2b0f.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-14T14:41:04,919 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 135a286d1aafce40676bb4d4370c9daf 2024-11-14T14:41:04,919 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d14b6f516265d1b90f14cf4519cf2b0f 2024-11-14T14:41:04,919 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated test,jjj,1731595263719.d14b6f516265d1b90f14cf4519cf2b0f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:41:04,919 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for d14b6f516265d1b90f14cf4519cf2b0f 2024-11-14T14:41:04,919 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for d14b6f516265d1b90f14cf4519cf2b0f 2024-11-14T14:41:04,920 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/135a286d1aafce40676bb4d4370c9daf 2024-11-14T14:41:04,921 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/135a286d1aafce40676bb4d4370c9daf 2024-11-14T14:41:04,922 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 135a286d1aafce40676bb4d4370c9daf 2024-11-14T14:41:04,922 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 135a286d1aafce40676bb4d4370c9daf 2024-11-14T14:41:04,922 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 550a3973a29751310a867312ba0b03d1, server=da1c6afcd1f9,46729,1731595261321 because future has completed 2024-11-14T14:41:04,924 INFO [StoreOpener-d14b6f516265d1b90f14cf4519cf2b0f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d14b6f516265d1b90f14cf4519cf2b0f 2024-11-14T14:41:04,925 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/9f8e87df66c884919328ba09109d819c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T14:41:04,926 INFO [StoreOpener-d14b6f516265d1b90f14cf4519cf2b0f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d14b6f516265d1b90f14cf4519cf2b0f columnFamilyName family 2024-11-14T14:41:04,926 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 135a286d1aafce40676bb4d4370c9daf 2024-11-14T14:41:04,926 DEBUG [StoreOpener-d14b6f516265d1b90f14cf4519cf2b0f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:41:04,927 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened 9f8e87df66c884919328ba09109d819c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63273756, jitterRate=-0.0571475625038147}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T14:41:04,927 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9f8e87df66c884919328ba09109d819c 2024-11-14T14:41:04,927 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for 9f8e87df66c884919328ba09109d819c: Running coprocessor pre-open hook at 1731595264899Writing region info on filesystem at 1731595264899Initializing all the Stores at 1731595264901 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595264901Cleaning up temporary data from old regions at 1731595264911 (+10 ms)Running coprocessor post-open hooks at 1731595264927 (+16 ms)Region opened successfully at 1731595264927 2024-11-14T14:41:04,928 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=19 2024-11-14T14:41:04,928 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=19, state=SUCCESS, hasLock=false; OpenRegionProcedure 550a3973a29751310a867312ba0b03d1, server=da1c6afcd1f9,46729,1731595261321 in 229 msec 2024-11-14T14:41:04,928 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1731595263719.9f8e87df66c884919328ba09109d819c., pid=43, masterSystemTime=1731595264828 2024-11-14T14:41:04,929 INFO [StoreOpener-d14b6f516265d1b90f14cf4519cf2b0f-1 {}] regionserver.HStore(327): Store=d14b6f516265d1b90f14cf4519cf2b0f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:41:04,929 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for d14b6f516265d1b90f14cf4519cf2b0f 2024-11-14T14:41:04,931 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/d14b6f516265d1b90f14cf4519cf2b0f 2024-11-14T14:41:04,931 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/d14b6f516265d1b90f14cf4519cf2b0f 2024-11-14T14:41:04,932 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1731595263719.9f8e87df66c884919328ba09109d819c. 2024-11-14T14:41:04,932 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened test,mmm,1731595263719.9f8e87df66c884919328ba09109d819c. 2024-11-14T14:41:04,932 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=550a3973a29751310a867312ba0b03d1, ASSIGN in 455 msec 2024-11-14T14:41:04,932 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for d14b6f516265d1b90f14cf4519cf2b0f 2024-11-14T14:41:04,932 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for d14b6f516265d1b90f14cf4519cf2b0f 2024-11-14T14:41:04,932 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open test,kkk,1731595263719.1d7e464b03c5060f7367637b64f2eae0. 2024-11-14T14:41:04,933 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => 1d7e464b03c5060f7367637b64f2eae0, NAME => 'test,kkk,1731595263719.1d7e464b03c5060f7367637b64f2eae0.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-11-14T14:41:04,933 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 1d7e464b03c5060f7367637b64f2eae0 2024-11-14T14:41:04,933 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated test,kkk,1731595263719.1d7e464b03c5060f7367637b64f2eae0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:41:04,934 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for 1d7e464b03c5060f7367637b64f2eae0 2024-11-14T14:41:04,934 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=9f8e87df66c884919328ba09109d819c, regionState=OPEN, openSeqNum=2, regionLocation=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:04,934 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for 1d7e464b03c5060f7367637b64f2eae0 2024-11-14T14:41:04,936 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for d14b6f516265d1b90f14cf4519cf2b0f 2024-11-14T14:41:04,937 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9f8e87df66c884919328ba09109d819c, server=da1c6afcd1f9,46229,1731595261363 because future has completed 2024-11-14T14:41:04,938 INFO [StoreOpener-1d7e464b03c5060f7367637b64f2eae0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 1d7e464b03c5060f7367637b64f2eae0 2024-11-14T14:41:04,940 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/135a286d1aafce40676bb4d4370c9daf/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T14:41:04,941 INFO [StoreOpener-1d7e464b03c5060f7367637b64f2eae0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1d7e464b03c5060f7367637b64f2eae0 columnFamilyName family 2024-11-14T14:41:04,941 DEBUG [StoreOpener-1d7e464b03c5060f7367637b64f2eae0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:41:04,941 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 135a286d1aafce40676bb4d4370c9daf; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74926369, jitterRate=0.11648990213871002}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T14:41:04,941 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 135a286d1aafce40676bb4d4370c9daf 2024-11-14T14:41:04,942 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/d14b6f516265d1b90f14cf4519cf2b0f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T14:41:04,942 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 135a286d1aafce40676bb4d4370c9daf: Running coprocessor pre-open hook at 1731595264910Writing region info on filesystem at 1731595264910Initializing all the Stores at 1731595264911 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595264911Cleaning up temporary data from old regions at 1731595264922 (+11 ms)Running coprocessor post-open hooks at 1731595264941 (+19 ms)Region opened successfully at 1731595264942 (+1 ms) 2024-11-14T14:41:04,942 INFO [StoreOpener-1d7e464b03c5060f7367637b64f2eae0-1 {}] regionserver.HStore(327): Store=1d7e464b03c5060f7367637b64f2eae0/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:41:04,942 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for 1d7e464b03c5060f7367637b64f2eae0 2024-11-14T14:41:04,943 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened d14b6f516265d1b90f14cf4519cf2b0f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60276631, jitterRate=-0.10180820524692535}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T14:41:04,943 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1731595263719.135a286d1aafce40676bb4d4370c9daf., pid=33, masterSystemTime=1731595264800 2024-11-14T14:41:04,943 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d14b6f516265d1b90f14cf4519cf2b0f 2024-11-14T14:41:04,943 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for d14b6f516265d1b90f14cf4519cf2b0f: Running coprocessor pre-open hook at 1731595264919Writing region info on filesystem at 1731595264919Initializing all the Stores at 1731595264921 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595264921Cleaning up temporary data from old regions at 1731595264932 (+11 ms)Running coprocessor post-open hooks at 1731595264943 (+11 ms)Region opened successfully at 1731595264943 2024-11-14T14:41:04,944 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/1d7e464b03c5060f7367637b64f2eae0 2024-11-14T14:41:04,945 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/1d7e464b03c5060f7367637b64f2eae0 2024-11-14T14:41:04,945 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for test,jjj,1731595263719.d14b6f516265d1b90f14cf4519cf2b0f., pid=53, masterSystemTime=1731595264906 2024-11-14T14:41:04,946 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for 1d7e464b03c5060f7367637b64f2eae0 2024-11-14T14:41:04,946 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for 1d7e464b03c5060f7367637b64f2eae0 2024-11-14T14:41:04,949 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for 1d7e464b03c5060f7367637b64f2eae0 2024-11-14T14:41:04,953 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=17 2024-11-14T14:41:04,953 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure 9f8e87df66c884919328ba09109d819c, server=da1c6afcd1f9,46229,1731595261363 in 212 msec 2024-11-14T14:41:04,953 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for test,iii,1731595263719.135a286d1aafce40676bb4d4370c9daf. 2024-11-14T14:41:04,954 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened test,iii,1731595263719.135a286d1aafce40676bb4d4370c9daf. 2024-11-14T14:41:04,954 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open test,lll,1731595263719.77f677c4c4cfcadb020d2436881cdeaa. 2024-11-14T14:41:04,954 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 77f677c4c4cfcadb020d2436881cdeaa, NAME => 'test,lll,1731595263719.77f677c4c4cfcadb020d2436881cdeaa.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-14T14:41:04,955 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 77f677c4c4cfcadb020d2436881cdeaa 2024-11-14T14:41:04,955 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=135a286d1aafce40676bb4d4370c9daf, regionState=OPEN, openSeqNum=2, regionLocation=da1c6afcd1f9,46729,1731595261321 2024-11-14T14:41:04,955 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated test,lll,1731595263719.77f677c4c4cfcadb020d2436881cdeaa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:41:04,955 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 77f677c4c4cfcadb020d2436881cdeaa 2024-11-14T14:41:04,955 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 77f677c4c4cfcadb020d2436881cdeaa 2024-11-14T14:41:04,956 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for test,jjj,1731595263719.d14b6f516265d1b90f14cf4519cf2b0f. 2024-11-14T14:41:04,956 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened test,jjj,1731595263719.d14b6f516265d1b90f14cf4519cf2b0f. 2024-11-14T14:41:04,957 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(132): Open test,xxx,1731595263719.954bd925c25bfdb6e075a7471bbb1c4d. 2024-11-14T14:41:04,957 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7752): Opening region: {ENCODED => 954bd925c25bfdb6e075a7471bbb1c4d, NAME => 'test,xxx,1731595263719.954bd925c25bfdb6e075a7471bbb1c4d.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-11-14T14:41:04,958 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 954bd925c25bfdb6e075a7471bbb1c4d 2024-11-14T14:41:04,958 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(898): Instantiated test,xxx,1731595263719.954bd925c25bfdb6e075a7471bbb1c4d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:41:04,958 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7794): checking encryption for 954bd925c25bfdb6e075a7471bbb1c4d 2024-11-14T14:41:04,959 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7797): checking classloading for 954bd925c25bfdb6e075a7471bbb1c4d 2024-11-14T14:41:04,959 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 135a286d1aafce40676bb4d4370c9daf, server=da1c6afcd1f9,46729,1731595261321 because future has completed 2024-11-14T14:41:04,960 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=d14b6f516265d1b90f14cf4519cf2b0f, regionState=OPEN, openSeqNum=2, regionLocation=da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:04,965 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/1d7e464b03c5060f7367637b64f2eae0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T14:41:04,966 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened 1d7e464b03c5060f7367637b64f2eae0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65283441, jitterRate=-0.02720092236995697}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T14:41:04,966 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1d7e464b03c5060f7367637b64f2eae0 2024-11-14T14:41:04,966 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for 1d7e464b03c5060f7367637b64f2eae0: Running coprocessor pre-open hook at 1731595264934Writing region info on filesystem at 1731595264934Initializing all the Stores at 1731595264936 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595264937 (+1 ms)Cleaning up temporary data from old regions at 1731595264946 (+9 ms)Running coprocessor post-open hooks at 1731595264966 (+20 ms)Region opened successfully at 1731595264966 2024-11-14T14:41:04,972 INFO [StoreOpener-77f677c4c4cfcadb020d2436881cdeaa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 77f677c4c4cfcadb020d2436881cdeaa 2024-11-14T14:41:04,972 INFO [StoreOpener-954bd925c25bfdb6e075a7471bbb1c4d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 954bd925c25bfdb6e075a7471bbb1c4d 2024-11-14T14:41:04,975 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure d14b6f516265d1b90f14cf4519cf2b0f, server=da1c6afcd1f9,40103,1731595261206 because future has completed 2024-11-14T14:41:04,975 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for test,kkk,1731595263719.1d7e464b03c5060f7367637b64f2eae0., pid=41, masterSystemTime=1731595264828 2024-11-14T14:41:04,977 INFO [StoreOpener-954bd925c25bfdb6e075a7471bbb1c4d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 954bd925c25bfdb6e075a7471bbb1c4d columnFamilyName family 2024-11-14T14:41:04,977 INFO [StoreOpener-77f677c4c4cfcadb020d2436881cdeaa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 77f677c4c4cfcadb020d2436881cdeaa columnFamilyName family 2024-11-14T14:41:04,977 DEBUG [StoreOpener-954bd925c25bfdb6e075a7471bbb1c4d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:41:04,977 DEBUG [StoreOpener-77f677c4c4cfcadb020d2436881cdeaa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:41:04,978 INFO [StoreOpener-77f677c4c4cfcadb020d2436881cdeaa-1 {}] regionserver.HStore(327): Store=77f677c4c4cfcadb020d2436881cdeaa/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:41:04,978 INFO [StoreOpener-954bd925c25bfdb6e075a7471bbb1c4d-1 {}] regionserver.HStore(327): Store=954bd925c25bfdb6e075a7471bbb1c4d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:41:04,978 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 77f677c4c4cfcadb020d2436881cdeaa 2024-11-14T14:41:04,979 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1038): replaying wal for 954bd925c25bfdb6e075a7471bbb1c4d 2024-11-14T14:41:04,979 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for test,kkk,1731595263719.1d7e464b03c5060f7367637b64f2eae0. 2024-11-14T14:41:04,979 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened test,kkk,1731595263719.1d7e464b03c5060f7367637b64f2eae0. 2024-11-14T14:41:04,979 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open test,www,1731595263719.dbc3f9fabaa1bd9fca0801fe6494782e. 2024-11-14T14:41:04,980 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => dbc3f9fabaa1bd9fca0801fe6494782e, NAME => 'test,www,1731595263719.dbc3f9fabaa1bd9fca0801fe6494782e.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-11-14T14:41:04,980 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/77f677c4c4cfcadb020d2436881cdeaa 2024-11-14T14:41:04,980 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/954bd925c25bfdb6e075a7471bbb1c4d 2024-11-14T14:41:04,981 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/77f677c4c4cfcadb020d2436881cdeaa 2024-11-14T14:41:04,981 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test dbc3f9fabaa1bd9fca0801fe6494782e 2024-11-14T14:41:04,981 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 77f677c4c4cfcadb020d2436881cdeaa 2024-11-14T14:41:04,982 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 77f677c4c4cfcadb020d2436881cdeaa 2024-11-14T14:41:04,982 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=13 2024-11-14T14:41:04,982 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated test,www,1731595263719.dbc3f9fabaa1bd9fca0801fe6494782e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:41:04,982 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure 135a286d1aafce40676bb4d4370c9daf, server=da1c6afcd1f9,46729,1731595261321 in 314 msec 2024-11-14T14:41:04,982 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for dbc3f9fabaa1bd9fca0801fe6494782e 2024-11-14T14:41:04,982 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for dbc3f9fabaa1bd9fca0801fe6494782e 2024-11-14T14:41:04,982 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/954bd925c25bfdb6e075a7471bbb1c4d 2024-11-14T14:41:04,983 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1048): stopping wal replay for 954bd925c25bfdb6e075a7471bbb1c4d 2024-11-14T14:41:04,984 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1060): Cleaning up temporary data for 954bd925c25bfdb6e075a7471bbb1c4d 2024-11-14T14:41:04,986 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 77f677c4c4cfcadb020d2436881cdeaa 2024-11-14T14:41:04,986 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=135a286d1aafce40676bb4d4370c9daf, ASSIGN in 509 msec 2024-11-14T14:41:04,987 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): writing seq id for 954bd925c25bfdb6e075a7471bbb1c4d 2024-11-14T14:41:04,988 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=1d7e464b03c5060f7367637b64f2eae0, regionState=OPEN, openSeqNum=2, regionLocation=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:04,992 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1d7e464b03c5060f7367637b64f2eae0, server=da1c6afcd1f9,46229,1731595261363 because future has completed 2024-11-14T14:41:04,998 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=14 2024-11-14T14:41:04,998 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure d14b6f516265d1b90f14cf4519cf2b0f, server=da1c6afcd1f9,40103,1731595261206 in 204 msec 2024-11-14T14:41:05,002 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d14b6f516265d1b90f14cf4519cf2b0f, ASSIGN in 525 msec 2024-11-14T14:41:05,006 INFO [StoreOpener-dbc3f9fabaa1bd9fca0801fe6494782e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region dbc3f9fabaa1bd9fca0801fe6494782e 2024-11-14T14:41:05,006 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/954bd925c25bfdb6e075a7471bbb1c4d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T14:41:05,006 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/77f677c4c4cfcadb020d2436881cdeaa/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T14:41:05,007 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1114): Opened 954bd925c25bfdb6e075a7471bbb1c4d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59077562, jitterRate=-0.11967572569847107}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T14:41:05,007 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 954bd925c25bfdb6e075a7471bbb1c4d 2024-11-14T14:41:05,008 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1006): Region open journal for 954bd925c25bfdb6e075a7471bbb1c4d: Running coprocessor pre-open hook at 1731595264959Writing region info on filesystem at 1731595264959Initializing all the Stores at 1731595264961 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595264961Cleaning up temporary data from old regions at 1731595264984 (+23 ms)Running coprocessor post-open hooks at 1731595265007 (+23 ms)Region opened successfully at 1731595265007 2024-11-14T14:41:05,008 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 77f677c4c4cfcadb020d2436881cdeaa; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68371564, jitterRate=0.018815696239471436}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T14:41:05,008 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 77f677c4c4cfcadb020d2436881cdeaa 2024-11-14T14:41:05,008 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 77f677c4c4cfcadb020d2436881cdeaa: Running coprocessor pre-open hook at 1731595264955Writing region info on filesystem at 1731595264955Initializing all the Stores at 1731595264959 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595264959Cleaning up temporary data from old regions at 1731595264982 (+23 ms)Running coprocessor post-open hooks at 1731595265008 (+26 ms)Region opened successfully at 1731595265008 2024-11-14T14:41:05,008 INFO [StoreOpener-dbc3f9fabaa1bd9fca0801fe6494782e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dbc3f9fabaa1bd9fca0801fe6494782e columnFamilyName family 2024-11-14T14:41:05,008 DEBUG [StoreOpener-dbc3f9fabaa1bd9fca0801fe6494782e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:41:05,009 INFO [StoreOpener-dbc3f9fabaa1bd9fca0801fe6494782e-1 {}] regionserver.HStore(327): Store=dbc3f9fabaa1bd9fca0801fe6494782e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:41:05,009 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for dbc3f9fabaa1bd9fca0801fe6494782e 2024-11-14T14:41:05,009 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1731595263719.77f677c4c4cfcadb020d2436881cdeaa., pid=37, masterSystemTime=1731595264800 2024-11-14T14:41:05,010 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1731595263719.954bd925c25bfdb6e075a7471bbb1c4d., pid=51, masterSystemTime=1731595264906 2024-11-14T14:41:05,011 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/dbc3f9fabaa1bd9fca0801fe6494782e 2024-11-14T14:41:05,011 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/dbc3f9fabaa1bd9fca0801fe6494782e 2024-11-14T14:41:05,012 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for dbc3f9fabaa1bd9fca0801fe6494782e 2024-11-14T14:41:05,012 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for dbc3f9fabaa1bd9fca0801fe6494782e 2024-11-14T14:41:05,013 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=15 2024-11-14T14:41:05,014 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure 1d7e464b03c5060f7367637b64f2eae0, server=da1c6afcd1f9,46229,1731595261363 in 287 msec 2024-11-14T14:41:05,014 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1731595263719.77f677c4c4cfcadb020d2436881cdeaa. 2024-11-14T14:41:05,014 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened test,lll,1731595263719.77f677c4c4cfcadb020d2436881cdeaa. 2024-11-14T14:41:05,015 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open test,bbb,1731595263719.b7030563781ca9d5e4d044b06d29712c. 2024-11-14T14:41:05,015 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => b7030563781ca9d5e4d044b06d29712c, NAME => 'test,bbb,1731595263719.b7030563781ca9d5e4d044b06d29712c.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-11-14T14:41:05,015 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test b7030563781ca9d5e4d044b06d29712c 2024-11-14T14:41:05,015 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated test,bbb,1731595263719.b7030563781ca9d5e4d044b06d29712c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:41:05,016 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for b7030563781ca9d5e4d044b06d29712c 2024-11-14T14:41:05,016 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for b7030563781ca9d5e4d044b06d29712c 2024-11-14T14:41:05,016 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for dbc3f9fabaa1bd9fca0801fe6494782e 2024-11-14T14:41:05,020 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=77f677c4c4cfcadb020d2436881cdeaa, regionState=OPEN, openSeqNum=2, regionLocation=da1c6afcd1f9,46729,1731595261321 2024-11-14T14:41:05,021 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1731595263719.954bd925c25bfdb6e075a7471bbb1c4d. 2024-11-14T14:41:05,021 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(153): Opened test,xxx,1731595263719.954bd925c25bfdb6e075a7471bbb1c4d. 2024-11-14T14:41:05,021 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(132): Open test,,1731595263719.3613f06851ef4465281b91596c2f4aa4. 2024-11-14T14:41:05,022 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7752): Opening region: {ENCODED => 3613f06851ef4465281b91596c2f4aa4, NAME => 'test,,1731595263719.3613f06851ef4465281b91596c2f4aa4.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-14T14:41:05,022 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 3613f06851ef4465281b91596c2f4aa4 2024-11-14T14:41:05,022 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(898): Instantiated test,,1731595263719.3613f06851ef4465281b91596c2f4aa4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:41:05,023 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7794): checking encryption for 3613f06851ef4465281b91596c2f4aa4 2024-11-14T14:41:05,023 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7797): checking classloading for 3613f06851ef4465281b91596c2f4aa4 2024-11-14T14:41:05,026 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 77f677c4c4cfcadb020d2436881cdeaa, server=da1c6afcd1f9,46729,1731595261321 because future has completed 2024-11-14T14:41:05,028 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=954bd925c25bfdb6e075a7471bbb1c4d, regionState=OPEN, openSeqNum=2, regionLocation=da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:05,030 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/dbc3f9fabaa1bd9fca0801fe6494782e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T14:41:05,032 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened dbc3f9fabaa1bd9fca0801fe6494782e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65713639, jitterRate=-0.020790472626686096}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T14:41:05,032 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for dbc3f9fabaa1bd9fca0801fe6494782e 2024-11-14T14:41:05,032 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for dbc3f9fabaa1bd9fca0801fe6494782e: Running coprocessor pre-open hook at 1731595264982Writing region info on filesystem at 1731595264982Initializing all the Stores at 1731595264986 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595264986Cleaning up temporary data from old regions at 1731595265012 (+26 ms)Running coprocessor post-open hooks at 1731595265032 (+20 ms)Region opened successfully at 1731595265032 2024-11-14T14:41:05,032 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 954bd925c25bfdb6e075a7471bbb1c4d, server=da1c6afcd1f9,40103,1731595261206 because future has completed 2024-11-14T14:41:05,033 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for test,www,1731595263719.dbc3f9fabaa1bd9fca0801fe6494782e., pid=40, masterSystemTime=1731595264828 2024-11-14T14:41:05,037 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for test,www,1731595263719.dbc3f9fabaa1bd9fca0801fe6494782e. 2024-11-14T14:41:05,037 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened test,www,1731595263719.dbc3f9fabaa1bd9fca0801fe6494782e. 2024-11-14T14:41:05,037 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(132): Open test,ccc,1731595263719.c1c1eeeedb7ad89130474f95f0b19f70. 2024-11-14T14:41:05,037 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7752): Opening region: {ENCODED => c1c1eeeedb7ad89130474f95f0b19f70, NAME => 'test,ccc,1731595263719.c1c1eeeedb7ad89130474f95f0b19f70.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-14T14:41:05,038 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c1c1eeeedb7ad89130474f95f0b19f70 2024-11-14T14:41:05,038 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(898): Instantiated test,ccc,1731595263719.c1c1eeeedb7ad89130474f95f0b19f70.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:41:05,038 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7794): checking encryption for c1c1eeeedb7ad89130474f95f0b19f70 2024-11-14T14:41:05,038 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7797): checking classloading for c1c1eeeedb7ad89130474f95f0b19f70 2024-11-14T14:41:05,039 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=16 2024-11-14T14:41:05,039 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 77f677c4c4cfcadb020d2436881cdeaa, server=da1c6afcd1f9,46729,1731595261321 in 344 msec 2024-11-14T14:41:05,040 INFO [StoreOpener-b7030563781ca9d5e4d044b06d29712c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region b7030563781ca9d5e4d044b06d29712c 2024-11-14T14:41:05,041 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN in 566 msec 2024-11-14T14:41:05,043 INFO [StoreOpener-b7030563781ca9d5e4d044b06d29712c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b7030563781ca9d5e4d044b06d29712c columnFamilyName family 2024-11-14T14:41:05,043 DEBUG [StoreOpener-b7030563781ca9d5e4d044b06d29712c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:41:05,044 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=dbc3f9fabaa1bd9fca0801fe6494782e, regionState=OPEN, openSeqNum=2, regionLocation=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:05,044 INFO [StoreOpener-b7030563781ca9d5e4d044b06d29712c-1 {}] regionserver.HStore(327): Store=b7030563781ca9d5e4d044b06d29712c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:41:05,044 INFO [StoreOpener-3613f06851ef4465281b91596c2f4aa4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 3613f06851ef4465281b91596c2f4aa4 2024-11-14T14:41:05,045 INFO [StoreOpener-c1c1eeeedb7ad89130474f95f0b19f70-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c1c1eeeedb7ad89130474f95f0b19f70 2024-11-14T14:41:05,044 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for b7030563781ca9d5e4d044b06d29712c 2024-11-14T14:41:05,047 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/b7030563781ca9d5e4d044b06d29712c 2024-11-14T14:41:05,048 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/b7030563781ca9d5e4d044b06d29712c 2024-11-14T14:41:05,049 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure dbc3f9fabaa1bd9fca0801fe6494782e, server=da1c6afcd1f9,46229,1731595261363 because future has completed 2024-11-14T14:41:05,049 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for b7030563781ca9d5e4d044b06d29712c 2024-11-14T14:41:05,049 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for b7030563781ca9d5e4d044b06d29712c 2024-11-14T14:41:05,050 INFO [StoreOpener-3613f06851ef4465281b91596c2f4aa4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3613f06851ef4465281b91596c2f4aa4 columnFamilyName family 2024-11-14T14:41:05,050 DEBUG [StoreOpener-3613f06851ef4465281b91596c2f4aa4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:41:05,051 INFO [StoreOpener-c1c1eeeedb7ad89130474f95f0b19f70-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1c1eeeedb7ad89130474f95f0b19f70 columnFamilyName family 2024-11-14T14:41:05,051 DEBUG [StoreOpener-c1c1eeeedb7ad89130474f95f0b19f70-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:41:05,052 INFO [StoreOpener-3613f06851ef4465281b91596c2f4aa4-1 {}] regionserver.HStore(327): Store=3613f06851ef4465281b91596c2f4aa4/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:41:05,052 INFO [StoreOpener-c1c1eeeedb7ad89130474f95f0b19f70-1 {}] regionserver.HStore(327): Store=c1c1eeeedb7ad89130474f95f0b19f70/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:41:05,052 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1038): replaying wal for 3613f06851ef4465281b91596c2f4aa4 2024-11-14T14:41:05,052 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1038): replaying wal for c1c1eeeedb7ad89130474f95f0b19f70 2024-11-14T14:41:05,054 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/c1c1eeeedb7ad89130474f95f0b19f70 2024-11-14T14:41:05,054 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/3613f06851ef4465281b91596c2f4aa4 2024-11-14T14:41:05,054 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=28 2024-11-14T14:41:05,054 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/3613f06851ef4465281b91596c2f4aa4 2024-11-14T14:41:05,054 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=28, state=SUCCESS, hasLock=false; OpenRegionProcedure 954bd925c25bfdb6e075a7471bbb1c4d, server=da1c6afcd1f9,40103,1731595261206 in 276 msec 2024-11-14T14:41:05,055 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/c1c1eeeedb7ad89130474f95f0b19f70 2024-11-14T14:41:05,055 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for b7030563781ca9d5e4d044b06d29712c 2024-11-14T14:41:05,055 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1048): stopping wal replay for 3613f06851ef4465281b91596c2f4aa4 2024-11-14T14:41:05,055 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1060): Cleaning up temporary data for 3613f06851ef4465281b91596c2f4aa4 2024-11-14T14:41:05,056 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1048): stopping wal replay for c1c1eeeedb7ad89130474f95f0b19f70 2024-11-14T14:41:05,056 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1060): Cleaning up temporary data for c1c1eeeedb7ad89130474f95f0b19f70 2024-11-14T14:41:05,058 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=954bd925c25bfdb6e075a7471bbb1c4d, ASSIGN in 582 msec 2024-11-14T14:41:05,061 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34057 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=da1c6afcd1f9,46229,1731595261363, table=test, region=dbc3f9fabaa1bd9fca0801fe6494782e. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-14T14:41:05,061 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1093): writing seq id for 3613f06851ef4465281b91596c2f4aa4 2024-11-14T14:41:05,064 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): writing seq id for c1c1eeeedb7ad89130474f95f0b19f70 2024-11-14T14:41:05,068 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=27 2024-11-14T14:41:05,068 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure dbc3f9fabaa1bd9fca0801fe6494782e, server=da1c6afcd1f9,46229,1731595261363 in 353 msec 2024-11-14T14:41:05,069 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/b7030563781ca9d5e4d044b06d29712c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T14:41:05,071 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/3613f06851ef4465281b91596c2f4aa4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T14:41:05,071 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened b7030563781ca9d5e4d044b06d29712c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59833686, jitterRate=-0.1084086000919342}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T14:41:05,071 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b7030563781ca9d5e4d044b06d29712c 2024-11-14T14:41:05,071 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for b7030563781ca9d5e4d044b06d29712c: Running coprocessor pre-open hook at 1731595265016Writing region info on filesystem at 1731595265016Initializing all the Stores at 1731595265020 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595265020Cleaning up temporary data from old regions at 1731595265050 (+30 ms)Running coprocessor post-open hooks at 1731595265071 (+21 ms)Region opened successfully at 1731595265071 2024-11-14T14:41:05,071 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=dbc3f9fabaa1bd9fca0801fe6494782e, ASSIGN in 595 msec 2024-11-14T14:41:05,073 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for test,bbb,1731595263719.b7030563781ca9d5e4d044b06d29712c., pid=35, masterSystemTime=1731595264800 2024-11-14T14:41:05,073 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1114): Opened 3613f06851ef4465281b91596c2f4aa4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63580206, jitterRate=-0.05258110165596008}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T14:41:05,074 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3613f06851ef4465281b91596c2f4aa4 2024-11-14T14:41:05,074 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1006): Region open journal for 3613f06851ef4465281b91596c2f4aa4: Running coprocessor pre-open hook at 1731595265023Writing region info on filesystem at 1731595265023Initializing all the Stores at 1731595265026 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595265026Cleaning up temporary data from old regions at 1731595265055 (+29 ms)Running coprocessor post-open hooks at 1731595265074 (+19 ms)Region opened successfully at 1731595265074 2024-11-14T14:41:05,076 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/c1c1eeeedb7ad89130474f95f0b19f70/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T14:41:05,076 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1731595263719.3613f06851ef4465281b91596c2f4aa4., pid=54, masterSystemTime=1731595264906 2024-11-14T14:41:05,077 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1114): Opened c1c1eeeedb7ad89130474f95f0b19f70; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65538202, jitterRate=-0.023404687643051147}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T14:41:05,077 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c1c1eeeedb7ad89130474f95f0b19f70 2024-11-14T14:41:05,077 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1006): Region open journal for c1c1eeeedb7ad89130474f95f0b19f70: Running coprocessor pre-open hook at 1731595265038Writing region info on filesystem at 1731595265038Initializing all the Stores at 1731595265040 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595265040Cleaning up temporary data from old regions at 1731595265056 (+16 ms)Running coprocessor post-open hooks at 1731595265077 (+21 ms)Region opened successfully at 1731595265077 2024-11-14T14:41:05,081 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1731595263719.c1c1eeeedb7ad89130474f95f0b19f70., pid=52, masterSystemTime=1731595264828 2024-11-14T14:41:05,082 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for test,bbb,1731595263719.b7030563781ca9d5e4d044b06d29712c. 2024-11-14T14:41:05,082 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened test,bbb,1731595263719.b7030563781ca9d5e4d044b06d29712c. 2024-11-14T14:41:05,082 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=1d7e464b03c5060f7367637b64f2eae0, ASSIGN in 598 msec 2024-11-14T14:41:05,084 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open test,yyy,1731595263719.76be481b709a73fdf12e38c88e45d041. 2024-11-14T14:41:05,084 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => 76be481b709a73fdf12e38c88e45d041, NAME => 'test,yyy,1731595263719.76be481b709a73fdf12e38c88e45d041.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-14T14:41:05,084 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 76be481b709a73fdf12e38c88e45d041 2024-11-14T14:41:05,085 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated test,yyy,1731595263719.76be481b709a73fdf12e38c88e45d041.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:41:05,085 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for 76be481b709a73fdf12e38c88e45d041 2024-11-14T14:41:05,085 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for 76be481b709a73fdf12e38c88e45d041 2024-11-14T14:41:05,088 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2266): Finished post open deploy task for test,,1731595263719.3613f06851ef4465281b91596c2f4aa4. 2024-11-14T14:41:05,089 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(153): Opened test,,1731595263719.3613f06851ef4465281b91596c2f4aa4. 2024-11-14T14:41:05,089 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(132): Open test,hhh,1731595263719.ad7ad94ee453836a534ae596990b42b8. 2024-11-14T14:41:05,089 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7752): Opening region: {ENCODED => ad7ad94ee453836a534ae596990b42b8, NAME => 'test,hhh,1731595263719.ad7ad94ee453836a534ae596990b42b8.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-14T14:41:05,089 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ad7ad94ee453836a534ae596990b42b8 2024-11-14T14:41:05,090 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(898): Instantiated test,hhh,1731595263719.ad7ad94ee453836a534ae596990b42b8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:41:05,090 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=b7030563781ca9d5e4d044b06d29712c, regionState=OPEN, openSeqNum=2, regionLocation=da1c6afcd1f9,46729,1731595261321 2024-11-14T14:41:05,090 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7794): checking encryption for ad7ad94ee453836a534ae596990b42b8 2024-11-14T14:41:05,090 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7797): checking classloading for ad7ad94ee453836a534ae596990b42b8 2024-11-14T14:41:05,090 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ccc,1731595263719.c1c1eeeedb7ad89130474f95f0b19f70. 2024-11-14T14:41:05,090 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(153): Opened test,ccc,1731595263719.c1c1eeeedb7ad89130474f95f0b19f70. 2024-11-14T14:41:05,091 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open test,zzz,1731595263719.1316438481dbf11d64fb665f6cdd4eb6. 2024-11-14T14:41:05,091 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => 1316438481dbf11d64fb665f6cdd4eb6, NAME => 'test,zzz,1731595263719.1316438481dbf11d64fb665f6cdd4eb6.', STARTKEY => 'zzz', ENDKEY => ''} 2024-11-14T14:41:05,091 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 1316438481dbf11d64fb665f6cdd4eb6 2024-11-14T14:41:05,091 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated test,zzz,1731595263719.1316438481dbf11d64fb665f6cdd4eb6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:41:05,092 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for 1316438481dbf11d64fb665f6cdd4eb6 2024-11-14T14:41:05,092 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for 1316438481dbf11d64fb665f6cdd4eb6 2024-11-14T14:41:05,095 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7030563781ca9d5e4d044b06d29712c, server=da1c6afcd1f9,46729,1731595261321 because future has completed 2024-11-14T14:41:05,101 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=6 2024-11-14T14:41:05,102 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure b7030563781ca9d5e4d044b06d29712c, server=da1c6afcd1f9,46729,1731595261321 in 419 msec 2024-11-14T14:41:05,104 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN in 629 msec 2024-11-14T14:41:05,105 INFO [StoreOpener-76be481b709a73fdf12e38c88e45d041-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 76be481b709a73fdf12e38c88e45d041 2024-11-14T14:41:05,106 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=c1c1eeeedb7ad89130474f95f0b19f70, regionState=OPEN, openSeqNum=2, regionLocation=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:05,108 INFO [StoreOpener-76be481b709a73fdf12e38c88e45d041-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 76be481b709a73fdf12e38c88e45d041 columnFamilyName family 2024-11-14T14:41:05,108 DEBUG [StoreOpener-76be481b709a73fdf12e38c88e45d041-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:41:05,110 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=3613f06851ef4465281b91596c2f4aa4, regionState=OPEN, openSeqNum=2, regionLocation=da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:05,110 INFO [StoreOpener-76be481b709a73fdf12e38c88e45d041-1 {}] regionserver.HStore(327): Store=76be481b709a73fdf12e38c88e45d041/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:41:05,110 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for 76be481b709a73fdf12e38c88e45d041 2024-11-14T14:41:05,110 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure c1c1eeeedb7ad89130474f95f0b19f70, server=da1c6afcd1f9,46229,1731595261363 because future has completed 2024-11-14T14:41:05,111 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/76be481b709a73fdf12e38c88e45d041 2024-11-14T14:41:05,112 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/76be481b709a73fdf12e38c88e45d041 2024-11-14T14:41:05,113 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for 76be481b709a73fdf12e38c88e45d041 2024-11-14T14:41:05,113 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for 76be481b709a73fdf12e38c88e45d041 2024-11-14T14:41:05,115 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3613f06851ef4465281b91596c2f4aa4, server=da1c6afcd1f9,40103,1731595261206 because future has completed 2024-11-14T14:41:05,116 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for 76be481b709a73fdf12e38c88e45d041 2024-11-14T14:41:05,124 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=5 2024-11-14T14:41:05,124 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 3613f06851ef4465281b91596c2f4aa4, server=da1c6afcd1f9,40103,1731595261206 in 322 msec 2024-11-14T14:41:05,128 INFO [StoreOpener-1316438481dbf11d64fb665f6cdd4eb6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 1316438481dbf11d64fb665f6cdd4eb6 2024-11-14T14:41:05,131 INFO [StoreOpener-1316438481dbf11d64fb665f6cdd4eb6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1316438481dbf11d64fb665f6cdd4eb6 columnFamilyName family 2024-11-14T14:41:05,131 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=3613f06851ef4465281b91596c2f4aa4, ASSIGN in 651 msec 2024-11-14T14:41:05,131 DEBUG [StoreOpener-1316438481dbf11d64fb665f6cdd4eb6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:41:05,132 INFO [StoreOpener-1316438481dbf11d64fb665f6cdd4eb6-1 {}] regionserver.HStore(327): Store=1316438481dbf11d64fb665f6cdd4eb6/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:41:05,133 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for 1316438481dbf11d64fb665f6cdd4eb6 2024-11-14T14:41:05,134 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/1316438481dbf11d64fb665f6cdd4eb6 2024-11-14T14:41:05,135 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/1316438481dbf11d64fb665f6cdd4eb6 2024-11-14T14:41:05,135 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for 1316438481dbf11d64fb665f6cdd4eb6 2024-11-14T14:41:05,136 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for 1316438481dbf11d64fb665f6cdd4eb6 2024-11-14T14:41:05,137 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=7 2024-11-14T14:41:05,137 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure c1c1eeeedb7ad89130474f95f0b19f70, server=da1c6afcd1f9,46229,1731595261363 in 354 msec 2024-11-14T14:41:05,139 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for 1316438481dbf11d64fb665f6cdd4eb6 2024-11-14T14:41:05,141 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=c1c1eeeedb7ad89130474f95f0b19f70, ASSIGN in 665 msec 2024-11-14T14:41:05,144 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=9f8e87df66c884919328ba09109d819c, ASSIGN in 668 msec 2024-11-14T14:41:05,144 INFO [StoreOpener-ad7ad94ee453836a534ae596990b42b8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ad7ad94ee453836a534ae596990b42b8 2024-11-14T14:41:05,146 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/76be481b709a73fdf12e38c88e45d041/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T14:41:05,146 INFO [StoreOpener-ad7ad94ee453836a534ae596990b42b8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ad7ad94ee453836a534ae596990b42b8 columnFamilyName family 2024-11-14T14:41:05,147 DEBUG [StoreOpener-ad7ad94ee453836a534ae596990b42b8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:41:05,147 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened 76be481b709a73fdf12e38c88e45d041; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72668999, jitterRate=0.08285246789455414}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T14:41:05,147 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 76be481b709a73fdf12e38c88e45d041 2024-11-14T14:41:05,147 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for 76be481b709a73fdf12e38c88e45d041: Running coprocessor pre-open hook at 1731595265085Writing region info on filesystem at 1731595265085Initializing all the Stores at 1731595265087 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595265087Cleaning up temporary data from old regions at 1731595265113 (+26 ms)Running coprocessor post-open hooks at 1731595265147 (+34 ms)Region opened successfully at 1731595265147 2024-11-14T14:41:05,147 INFO [StoreOpener-ad7ad94ee453836a534ae596990b42b8-1 {}] regionserver.HStore(327): Store=ad7ad94ee453836a534ae596990b42b8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:41:05,147 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1038): replaying wal for ad7ad94ee453836a534ae596990b42b8 2024-11-14T14:41:05,148 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/1316438481dbf11d64fb665f6cdd4eb6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T14:41:05,150 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1731595263719.76be481b709a73fdf12e38c88e45d041., pid=31, masterSystemTime=1731595264800 2024-11-14T14:41:05,150 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/ad7ad94ee453836a534ae596990b42b8 2024-11-14T14:41:05,150 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened 1316438481dbf11d64fb665f6cdd4eb6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70701148, jitterRate=0.053529202938079834}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T14:41:05,150 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1316438481dbf11d64fb665f6cdd4eb6 2024-11-14T14:41:05,150 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for 1316438481dbf11d64fb665f6cdd4eb6: Running coprocessor pre-open hook at 1731595265092Writing region info on filesystem at 1731595265092Initializing all the Stores at 1731595265094 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595265094Cleaning up temporary data from old regions at 1731595265136 (+42 ms)Running coprocessor post-open hooks at 1731595265150 (+14 ms)Region opened successfully at 1731595265150 2024-11-14T14:41:05,151 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/ad7ad94ee453836a534ae596990b42b8 2024-11-14T14:41:05,151 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for test,zzz,1731595263719.1316438481dbf11d64fb665f6cdd4eb6., pid=34, masterSystemTime=1731595264828 2024-11-14T14:41:05,152 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1048): stopping wal replay for ad7ad94ee453836a534ae596990b42b8 2024-11-14T14:41:05,152 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1060): Cleaning up temporary data for ad7ad94ee453836a534ae596990b42b8 2024-11-14T14:41:05,154 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1731595263719.76be481b709a73fdf12e38c88e45d041. 2024-11-14T14:41:05,154 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened test,yyy,1731595263719.76be481b709a73fdf12e38c88e45d041. 2024-11-14T14:41:05,154 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open test,vvv,1731595263719.2e2ab5762f8d8f31683b4f4994048f3e. 2024-11-14T14:41:05,154 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => 2e2ab5762f8d8f31683b4f4994048f3e, NAME => 'test,vvv,1731595263719.2e2ab5762f8d8f31683b4f4994048f3e.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-14T14:41:05,155 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1093): writing seq id for ad7ad94ee453836a534ae596990b42b8 2024-11-14T14:41:05,155 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 2e2ab5762f8d8f31683b4f4994048f3e 2024-11-14T14:41:05,155 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated test,vvv,1731595263719.2e2ab5762f8d8f31683b4f4994048f3e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:41:05,155 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for 2e2ab5762f8d8f31683b4f4994048f3e 2024-11-14T14:41:05,155 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for 2e2ab5762f8d8f31683b4f4994048f3e 2024-11-14T14:41:05,156 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=76be481b709a73fdf12e38c88e45d041, regionState=OPEN, openSeqNum=2, regionLocation=da1c6afcd1f9,46729,1731595261321 2024-11-14T14:41:05,156 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for test,zzz,1731595263719.1316438481dbf11d64fb665f6cdd4eb6. 2024-11-14T14:41:05,156 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened test,zzz,1731595263719.1316438481dbf11d64fb665f6cdd4eb6. 2024-11-14T14:41:05,156 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open test,ttt,1731595263719.71350ab931ae29ff2210bd9d6c3eeac5. 2024-11-14T14:41:05,157 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => 71350ab931ae29ff2210bd9d6c3eeac5, NAME => 'test,ttt,1731595263719.71350ab931ae29ff2210bd9d6c3eeac5.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-14T14:41:05,157 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 71350ab931ae29ff2210bd9d6c3eeac5 2024-11-14T14:41:05,157 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated test,ttt,1731595263719.71350ab931ae29ff2210bd9d6c3eeac5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:41:05,157 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for 71350ab931ae29ff2210bd9d6c3eeac5 2024-11-14T14:41:05,158 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for 71350ab931ae29ff2210bd9d6c3eeac5 2024-11-14T14:41:05,161 INFO [StoreOpener-2e2ab5762f8d8f31683b4f4994048f3e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 2e2ab5762f8d8f31683b4f4994048f3e 2024-11-14T14:41:05,161 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=1316438481dbf11d64fb665f6cdd4eb6, regionState=OPEN, openSeqNum=2, regionLocation=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:05,162 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 76be481b709a73fdf12e38c88e45d041, server=da1c6afcd1f9,46729,1731595261321 because future has completed 2024-11-14T14:41:05,163 INFO [StoreOpener-2e2ab5762f8d8f31683b4f4994048f3e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2e2ab5762f8d8f31683b4f4994048f3e columnFamilyName family 2024-11-14T14:41:05,163 DEBUG [StoreOpener-2e2ab5762f8d8f31683b4f4994048f3e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:41:05,164 INFO [StoreOpener-2e2ab5762f8d8f31683b4f4994048f3e-1 {}] regionserver.HStore(327): Store=2e2ab5762f8d8f31683b4f4994048f3e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:41:05,165 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for 2e2ab5762f8d8f31683b4f4994048f3e 2024-11-14T14:41:05,166 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/2e2ab5762f8d8f31683b4f4994048f3e 2024-11-14T14:41:05,167 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/2e2ab5762f8d8f31683b4f4994048f3e 2024-11-14T14:41:05,167 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/ad7ad94ee453836a534ae596990b42b8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T14:41:05,168 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for 2e2ab5762f8d8f31683b4f4994048f3e 2024-11-14T14:41:05,168 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1114): Opened ad7ad94ee453836a534ae596990b42b8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73052299, jitterRate=0.08856408298015594}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T14:41:05,168 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ad7ad94ee453836a534ae596990b42b8 2024-11-14T14:41:05,168 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for 2e2ab5762f8d8f31683b4f4994048f3e 2024-11-14T14:41:05,168 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1006): Region open journal for ad7ad94ee453836a534ae596990b42b8: Running coprocessor pre-open hook at 1731595265091Writing region info on filesystem at 1731595265091Initializing all the Stores at 1731595265094 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595265094Cleaning up temporary data from old regions at 1731595265152 (+58 ms)Running coprocessor post-open hooks at 1731595265168 (+16 ms)Region opened successfully at 1731595265168 2024-11-14T14:41:05,168 INFO [StoreOpener-71350ab931ae29ff2210bd9d6c3eeac5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 71350ab931ae29ff2210bd9d6c3eeac5 2024-11-14T14:41:05,170 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1316438481dbf11d64fb665f6cdd4eb6, server=da1c6afcd1f9,46229,1731595261363 because future has completed 2024-11-14T14:41:05,170 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1731595263719.ad7ad94ee453836a534ae596990b42b8., pid=56, masterSystemTime=1731595264906 2024-11-14T14:41:05,171 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for 2e2ab5762f8d8f31683b4f4994048f3e 2024-11-14T14:41:05,174 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/2e2ab5762f8d8f31683b4f4994048f3e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T14:41:05,175 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened 2e2ab5762f8d8f31683b4f4994048f3e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59223426, jitterRate=-0.11750218272209167}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T14:41:05,175 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2e2ab5762f8d8f31683b4f4994048f3e 2024-11-14T14:41:05,175 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for 2e2ab5762f8d8f31683b4f4994048f3e: Running coprocessor pre-open hook at 1731595265156Writing region info on filesystem at 1731595265156Initializing all the Stores at 1731595265159 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595265159Cleaning up temporary data from old regions at 1731595265168 (+9 ms)Running coprocessor post-open hooks at 1731595265175 (+7 ms)Region opened successfully at 1731595265175 2024-11-14T14:41:05,177 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1731595263719.2e2ab5762f8d8f31683b4f4994048f3e., pid=39, masterSystemTime=1731595264800 2024-11-14T14:41:05,182 INFO [StoreOpener-71350ab931ae29ff2210bd9d6c3eeac5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 71350ab931ae29ff2210bd9d6c3eeac5 columnFamilyName family 2024-11-14T14:41:05,182 DEBUG [StoreOpener-71350ab931ae29ff2210bd9d6c3eeac5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:41:05,183 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2266): Finished post open deploy task for test,hhh,1731595263719.ad7ad94ee453836a534ae596990b42b8. 2024-11-14T14:41:05,183 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(153): Opened test,hhh,1731595263719.ad7ad94ee453836a534ae596990b42b8. 2024-11-14T14:41:05,183 INFO [StoreOpener-71350ab931ae29ff2210bd9d6c3eeac5-1 {}] regionserver.HStore(327): Store=71350ab931ae29ff2210bd9d6c3eeac5/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:41:05,183 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open test,uuu,1731595263719.6d725ecb42ad6f19c89d6ddaa229edc8. 2024-11-14T14:41:05,184 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for 71350ab931ae29ff2210bd9d6c3eeac5 2024-11-14T14:41:05,184 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => 6d725ecb42ad6f19c89d6ddaa229edc8, NAME => 'test,uuu,1731595263719.6d725ecb42ad6f19c89d6ddaa229edc8.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-14T14:41:05,184 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 6d725ecb42ad6f19c89d6ddaa229edc8 2024-11-14T14:41:05,184 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated test,uuu,1731595263719.6d725ecb42ad6f19c89d6ddaa229edc8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:41:05,184 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for 6d725ecb42ad6f19c89d6ddaa229edc8 2024-11-14T14:41:05,184 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for 6d725ecb42ad6f19c89d6ddaa229edc8 2024-11-14T14:41:05,185 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/71350ab931ae29ff2210bd9d6c3eeac5 2024-11-14T14:41:05,186 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34057 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=da1c6afcd1f9,46729,1731595261321, table=test, region=76be481b709a73fdf12e38c88e45d041. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-14T14:41:05,187 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/71350ab931ae29ff2210bd9d6c3eeac5 2024-11-14T14:41:05,188 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=30 2024-11-14T14:41:05,188 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure 1316438481dbf11d64fb665f6cdd4eb6, server=da1c6afcd1f9,46229,1731595261363 in 504 msec 2024-11-14T14:41:05,188 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1731595263719.2e2ab5762f8d8f31683b4f4994048f3e. 2024-11-14T14:41:05,188 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for 71350ab931ae29ff2210bd9d6c3eeac5 2024-11-14T14:41:05,188 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for 71350ab931ae29ff2210bd9d6c3eeac5 2024-11-14T14:41:05,188 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened test,vvv,1731595263719.2e2ab5762f8d8f31683b4f4994048f3e. 2024-11-14T14:41:05,191 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for 71350ab931ae29ff2210bd9d6c3eeac5 2024-11-14T14:41:05,191 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=1316438481dbf11d64fb665f6cdd4eb6, ASSIGN in 716 msec 2024-11-14T14:41:05,192 INFO [StoreOpener-6d725ecb42ad6f19c89d6ddaa229edc8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 6d725ecb42ad6f19c89d6ddaa229edc8 2024-11-14T14:41:05,193 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=2e2ab5762f8d8f31683b4f4994048f3e, regionState=OPEN, openSeqNum=2, regionLocation=da1c6afcd1f9,46729,1731595261321 2024-11-14T14:41:05,196 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/71350ab931ae29ff2210bd9d6c3eeac5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T14:41:05,197 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=ad7ad94ee453836a534ae596990b42b8, regionState=OPEN, openSeqNum=2, regionLocation=da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:05,197 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,46729,1731595261321 because future has completed 2024-11-14T14:41:05,197 INFO [StoreOpener-6d725ecb42ad6f19c89d6ddaa229edc8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6d725ecb42ad6f19c89d6ddaa229edc8 columnFamilyName family 2024-11-14T14:41:05,197 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened 71350ab931ae29ff2210bd9d6c3eeac5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59705354, jitterRate=-0.11032089591026306}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T14:41:05,197 DEBUG [StoreOpener-6d725ecb42ad6f19c89d6ddaa229edc8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:41:05,197 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 71350ab931ae29ff2210bd9d6c3eeac5 2024-11-14T14:41:05,197 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for 71350ab931ae29ff2210bd9d6c3eeac5: Running coprocessor pre-open hook at 1731595265158Writing region info on filesystem at 1731595265158Initializing all the Stores at 1731595265160 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595265160Cleaning up temporary data from old regions at 1731595265188 (+28 ms)Running coprocessor post-open hooks at 1731595265197 (+9 ms)Region opened successfully at 1731595265197 2024-11-14T14:41:05,198 INFO [StoreOpener-6d725ecb42ad6f19c89d6ddaa229edc8-1 {}] regionserver.HStore(327): Store=6d725ecb42ad6f19c89d6ddaa229edc8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:41:05,198 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for 6d725ecb42ad6f19c89d6ddaa229edc8 2024-11-14T14:41:05,199 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1731595263719.71350ab931ae29ff2210bd9d6c3eeac5., pid=45, masterSystemTime=1731595264828 2024-11-14T14:41:05,199 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/6d725ecb42ad6f19c89d6ddaa229edc8 2024-11-14T14:41:05,200 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure ad7ad94ee453836a534ae596990b42b8, server=da1c6afcd1f9,40103,1731595261206 because future has completed 2024-11-14T14:41:05,200 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/6d725ecb42ad6f19c89d6ddaa229edc8 2024-11-14T14:41:05,201 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for 6d725ecb42ad6f19c89d6ddaa229edc8 2024-11-14T14:41:05,201 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for 6d725ecb42ad6f19c89d6ddaa229edc8 2024-11-14T14:41:05,202 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1731595263719.71350ab931ae29ff2210bd9d6c3eeac5. 2024-11-14T14:41:05,202 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened test,ttt,1731595263719.71350ab931ae29ff2210bd9d6c3eeac5. 2024-11-14T14:41:05,202 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(132): Open test,ggg,1731595263719.9cb318690a770d1645dfd64bec1a284b. 2024-11-14T14:41:05,202 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7752): Opening region: {ENCODED => 9cb318690a770d1645dfd64bec1a284b, NAME => 'test,ggg,1731595263719.9cb318690a770d1645dfd64bec1a284b.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-11-14T14:41:05,203 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 9cb318690a770d1645dfd64bec1a284b 2024-11-14T14:41:05,203 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(898): Instantiated test,ggg,1731595263719.9cb318690a770d1645dfd64bec1a284b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:41:05,203 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7794): checking encryption for 9cb318690a770d1645dfd64bec1a284b 2024-11-14T14:41:05,204 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7797): checking classloading for 9cb318690a770d1645dfd64bec1a284b 2024-11-14T14:41:05,204 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for 6d725ecb42ad6f19c89d6ddaa229edc8 2024-11-14T14:41:05,206 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=26 2024-11-14T14:41:05,206 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,46729,1731595261321 in 501 msec 2024-11-14T14:41:05,207 INFO [StoreOpener-9cb318690a770d1645dfd64bec1a284b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 9cb318690a770d1645dfd64bec1a284b 2024-11-14T14:41:05,208 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/6d725ecb42ad6f19c89d6ddaa229edc8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T14:41:05,209 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=2e2ab5762f8d8f31683b4f4994048f3e, ASSIGN in 733 msec 2024-11-14T14:41:05,209 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened 6d725ecb42ad6f19c89d6ddaa229edc8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74688850, jitterRate=0.11295059323310852}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T14:41:05,209 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6d725ecb42ad6f19c89d6ddaa229edc8 2024-11-14T14:41:05,209 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for 6d725ecb42ad6f19c89d6ddaa229edc8: Running coprocessor pre-open hook at 1731595265185Writing region info on filesystem at 1731595265185Initializing all the Stores at 1731595265187 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595265187Cleaning up temporary data from old regions at 1731595265201 (+14 ms)Running coprocessor post-open hooks at 1731595265209 (+8 ms)Region opened successfully at 1731595265209 2024-11-14T14:41:05,210 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=71350ab931ae29ff2210bd9d6c3eeac5, regionState=OPEN, openSeqNum=2, regionLocation=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:05,210 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for test,uuu,1731595263719.6d725ecb42ad6f19c89d6ddaa229edc8., pid=47, masterSystemTime=1731595264906 2024-11-14T14:41:05,210 INFO [StoreOpener-9cb318690a770d1645dfd64bec1a284b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9cb318690a770d1645dfd64bec1a284b columnFamilyName family 2024-11-14T14:41:05,211 DEBUG [StoreOpener-9cb318690a770d1645dfd64bec1a284b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:41:05,211 INFO [StoreOpener-9cb318690a770d1645dfd64bec1a284b-1 {}] regionserver.HStore(327): Store=9cb318690a770d1645dfd64bec1a284b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:41:05,212 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1038): replaying wal for 9cb318690a770d1645dfd64bec1a284b 2024-11-14T14:41:05,213 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/9cb318690a770d1645dfd64bec1a284b 2024-11-14T14:41:05,214 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 71350ab931ae29ff2210bd9d6c3eeac5, server=da1c6afcd1f9,46229,1731595261363 because future has completed 2024-11-14T14:41:05,214 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for test,uuu,1731595263719.6d725ecb42ad6f19c89d6ddaa229edc8. 2024-11-14T14:41:05,214 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened test,uuu,1731595263719.6d725ecb42ad6f19c89d6ddaa229edc8. 2024-11-14T14:41:05,214 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/9cb318690a770d1645dfd64bec1a284b 2024-11-14T14:41:05,214 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(132): Open test,nnn,1731595263719.f0bff83267f1352a4c903f322542cf14. 2024-11-14T14:41:05,214 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7752): Opening region: {ENCODED => f0bff83267f1352a4c903f322542cf14, NAME => 'test,nnn,1731595263719.f0bff83267f1352a4c903f322542cf14.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-11-14T14:41:05,215 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f0bff83267f1352a4c903f322542cf14 2024-11-14T14:41:05,215 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(898): Instantiated test,nnn,1731595263719.f0bff83267f1352a4c903f322542cf14.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:41:05,215 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7794): checking encryption for f0bff83267f1352a4c903f322542cf14 2024-11-14T14:41:05,215 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7797): checking classloading for f0bff83267f1352a4c903f322542cf14 2024-11-14T14:41:05,216 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1048): stopping wal replay for 9cb318690a770d1645dfd64bec1a284b 2024-11-14T14:41:05,216 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1060): Cleaning up temporary data for 9cb318690a770d1645dfd64bec1a284b 2024-11-14T14:41:05,218 INFO [StoreOpener-f0bff83267f1352a4c903f322542cf14-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f0bff83267f1352a4c903f322542cf14 2024-11-14T14:41:05,219 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1093): writing seq id for 9cb318690a770d1645dfd64bec1a284b 2024-11-14T14:41:05,219 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=12 2024-11-14T14:41:05,219 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure ad7ad94ee453836a534ae596990b42b8, server=da1c6afcd1f9,40103,1731595261206 in 405 msec 2024-11-14T14:41:05,221 INFO [StoreOpener-f0bff83267f1352a4c903f322542cf14-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f0bff83267f1352a4c903f322542cf14 columnFamilyName family 2024-11-14T14:41:05,222 DEBUG [StoreOpener-f0bff83267f1352a4c903f322542cf14-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:41:05,222 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/9cb318690a770d1645dfd64bec1a284b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T14:41:05,223 INFO [StoreOpener-f0bff83267f1352a4c903f322542cf14-1 {}] regionserver.HStore(327): Store=f0bff83267f1352a4c903f322542cf14/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:41:05,223 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1038): replaying wal for f0bff83267f1352a4c903f322542cf14 2024-11-14T14:41:05,223 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1114): Opened 9cb318690a770d1645dfd64bec1a284b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68743585, jitterRate=0.024359241127967834}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T14:41:05,223 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9cb318690a770d1645dfd64bec1a284b 2024-11-14T14:41:05,223 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1006): Region open journal for 9cb318690a770d1645dfd64bec1a284b: Running coprocessor pre-open hook at 1731595265204Writing region info on filesystem at 1731595265204Initializing all the Stores at 1731595265206 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595265206Cleaning up temporary data from old regions at 1731595265216 (+10 ms)Running coprocessor post-open hooks at 1731595265223 (+7 ms)Region opened successfully at 1731595265223 2024-11-14T14:41:05,224 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=ad7ad94ee453836a534ae596990b42b8, ASSIGN in 746 msec 2024-11-14T14:41:05,224 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/f0bff83267f1352a4c903f322542cf14 2024-11-14T14:41:05,225 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/f0bff83267f1352a4c903f322542cf14 2024-11-14T14:41:05,226 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ggg,1731595263719.9cb318690a770d1645dfd64bec1a284b., pid=42, masterSystemTime=1731595264828 2024-11-14T14:41:05,226 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1048): stopping wal replay for f0bff83267f1352a4c903f322542cf14 2024-11-14T14:41:05,226 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1060): Cleaning up temporary data for f0bff83267f1352a4c903f322542cf14 2024-11-14T14:41:05,229 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1093): writing seq id for f0bff83267f1352a4c903f322542cf14 2024-11-14T14:41:05,229 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ggg,1731595263719.9cb318690a770d1645dfd64bec1a284b. 2024-11-14T14:41:05,229 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(153): Opened test,ggg,1731595263719.9cb318690a770d1645dfd64bec1a284b. 2024-11-14T14:41:05,231 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=24 2024-11-14T14:41:05,231 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure 71350ab931ae29ff2210bd9d6c3eeac5, server=da1c6afcd1f9,46229,1731595261363 in 490 msec 2024-11-14T14:41:05,232 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/f0bff83267f1352a4c903f322542cf14/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T14:41:05,233 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1114): Opened f0bff83267f1352a4c903f322542cf14; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71549536, jitterRate=0.06617116928100586}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T14:41:05,233 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f0bff83267f1352a4c903f322542cf14 2024-11-14T14:41:05,233 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1006): Region open journal for f0bff83267f1352a4c903f322542cf14: Running coprocessor pre-open hook at 1731595265215Writing region info on filesystem at 1731595265215Initializing all the Stores at 1731595265217 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595265217Cleaning up temporary data from old regions at 1731595265226 (+9 ms)Running coprocessor post-open hooks at 1731595265233 (+7 ms)Region opened successfully at 1731595265233 2024-11-14T14:41:05,234 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=71350ab931ae29ff2210bd9d6c3eeac5, ASSIGN in 758 msec 2024-11-14T14:41:05,235 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2236): Post open deploy tasks for test,nnn,1731595263719.f0bff83267f1352a4c903f322542cf14., pid=48, masterSystemTime=1731595264906 2024-11-14T14:41:05,235 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=9cb318690a770d1645dfd64bec1a284b, regionState=OPEN, openSeqNum=2, regionLocation=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:05,238 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2266): Finished post open deploy task for test,nnn,1731595263719.f0bff83267f1352a4c903f322542cf14. 2024-11-14T14:41:05,238 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(153): Opened test,nnn,1731595263719.f0bff83267f1352a4c903f322542cf14. 2024-11-14T14:41:05,238 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(132): Open test,rrr,1731595263719.b2f6569935d4c7faa1470e22f0b79a20. 2024-11-14T14:41:05,238 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9cb318690a770d1645dfd64bec1a284b, server=da1c6afcd1f9,46229,1731595261363 because future has completed 2024-11-14T14:41:05,239 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=f0bff83267f1352a4c903f322542cf14, regionState=OPEN, openSeqNum=2, regionLocation=da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:05,239 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7752): Opening region: {ENCODED => b2f6569935d4c7faa1470e22f0b79a20, NAME => 'test,rrr,1731595263719.b2f6569935d4c7faa1470e22f0b79a20.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-14T14:41:05,239 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test b2f6569935d4c7faa1470e22f0b79a20 2024-11-14T14:41:05,239 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(898): Instantiated test,rrr,1731595263719.b2f6569935d4c7faa1470e22f0b79a20.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:41:05,239 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7794): checking encryption for b2f6569935d4c7faa1470e22f0b79a20 2024-11-14T14:41:05,239 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7797): checking classloading for b2f6569935d4c7faa1470e22f0b79a20 2024-11-14T14:41:05,242 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=48, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure f0bff83267f1352a4c903f322542cf14, server=da1c6afcd1f9,40103,1731595261206 because future has completed 2024-11-14T14:41:05,242 INFO [StoreOpener-b2f6569935d4c7faa1470e22f0b79a20-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region b2f6569935d4c7faa1470e22f0b79a20 2024-11-14T14:41:05,244 INFO [StoreOpener-b2f6569935d4c7faa1470e22f0b79a20-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b2f6569935d4c7faa1470e22f0b79a20 columnFamilyName family 2024-11-14T14:41:05,245 DEBUG [StoreOpener-b2f6569935d4c7faa1470e22f0b79a20-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:41:05,245 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=11 2024-11-14T14:41:05,245 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 9cb318690a770d1645dfd64bec1a284b, server=da1c6afcd1f9,46229,1731595261363 in 520 msec 2024-11-14T14:41:05,245 INFO [StoreOpener-b2f6569935d4c7faa1470e22f0b79a20-1 {}] regionserver.HStore(327): Store=b2f6569935d4c7faa1470e22f0b79a20/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:41:05,245 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1038): replaying wal for b2f6569935d4c7faa1470e22f0b79a20 2024-11-14T14:41:05,246 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/b2f6569935d4c7faa1470e22f0b79a20 2024-11-14T14:41:05,247 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/b2f6569935d4c7faa1470e22f0b79a20 2024-11-14T14:41:05,247 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1048): stopping wal replay for b2f6569935d4c7faa1470e22f0b79a20 2024-11-14T14:41:05,247 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1060): Cleaning up temporary data for b2f6569935d4c7faa1470e22f0b79a20 2024-11-14T14:41:05,247 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=9cb318690a770d1645dfd64bec1a284b, ASSIGN in 772 msec 2024-11-14T14:41:05,250 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1093): writing seq id for b2f6569935d4c7faa1470e22f0b79a20 2024-11-14T14:41:05,252 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=18 2024-11-14T14:41:05,252 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure f0bff83267f1352a4c903f322542cf14, server=da1c6afcd1f9,40103,1731595261206 in 489 msec 2024-11-14T14:41:05,253 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/b2f6569935d4c7faa1470e22f0b79a20/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T14:41:05,254 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1114): Opened b2f6569935d4c7faa1470e22f0b79a20; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61316192, jitterRate=-0.08631753921508789}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T14:41:05,254 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b2f6569935d4c7faa1470e22f0b79a20 2024-11-14T14:41:05,254 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1006): Region open journal for b2f6569935d4c7faa1470e22f0b79a20: Running coprocessor pre-open hook at 1731595265240Writing region info on filesystem at 1731595265240Initializing all the Stores at 1731595265242 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595265242Cleaning up temporary data from old regions at 1731595265247 (+5 ms)Running coprocessor post-open hooks at 1731595265254 (+7 ms)Region opened successfully at 1731595265254 2024-11-14T14:41:05,255 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f0bff83267f1352a4c903f322542cf14, ASSIGN in 780 msec 2024-11-14T14:41:05,255 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1731595263719.b2f6569935d4c7faa1470e22f0b79a20., pid=50, masterSystemTime=1731595264906 2024-11-14T14:41:05,257 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=6d725ecb42ad6f19c89d6ddaa229edc8, regionState=OPEN, openSeqNum=2, regionLocation=da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:05,259 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1731595263719.b2f6569935d4c7faa1470e22f0b79a20. 2024-11-14T14:41:05,259 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(153): Opened test,rrr,1731595263719.b2f6569935d4c7faa1470e22f0b79a20. 2024-11-14T14:41:05,259 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(132): Open test,qqq,1731595263719.e1484a67e6b0eaf08cc5e1d6301f8319. 2024-11-14T14:41:05,259 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7752): Opening region: {ENCODED => e1484a67e6b0eaf08cc5e1d6301f8319, NAME => 'test,qqq,1731595263719.e1484a67e6b0eaf08cc5e1d6301f8319.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-11-14T14:41:05,260 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test e1484a67e6b0eaf08cc5e1d6301f8319 2024-11-14T14:41:05,260 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(898): Instantiated test,qqq,1731595263719.e1484a67e6b0eaf08cc5e1d6301f8319.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:41:05,260 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7794): checking encryption for e1484a67e6b0eaf08cc5e1d6301f8319 2024-11-14T14:41:05,260 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7797): checking classloading for e1484a67e6b0eaf08cc5e1d6301f8319 2024-11-14T14:41:05,262 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=b2f6569935d4c7faa1470e22f0b79a20, regionState=OPEN, openSeqNum=2, regionLocation=da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:05,262 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6d725ecb42ad6f19c89d6ddaa229edc8, server=da1c6afcd1f9,40103,1731595261206 because future has completed 2024-11-14T14:41:05,263 INFO [StoreOpener-e1484a67e6b0eaf08cc5e1d6301f8319-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region e1484a67e6b0eaf08cc5e1d6301f8319 2024-11-14T14:41:05,265 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure b2f6569935d4c7faa1470e22f0b79a20, server=da1c6afcd1f9,40103,1731595261206 because future has completed 2024-11-14T14:41:05,265 INFO [StoreOpener-e1484a67e6b0eaf08cc5e1d6301f8319-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e1484a67e6b0eaf08cc5e1d6301f8319 columnFamilyName family 2024-11-14T14:41:05,265 DEBUG [StoreOpener-e1484a67e6b0eaf08cc5e1d6301f8319-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:41:05,266 INFO [StoreOpener-e1484a67e6b0eaf08cc5e1d6301f8319-1 {}] regionserver.HStore(327): Store=e1484a67e6b0eaf08cc5e1d6301f8319/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:41:05,266 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1038): replaying wal for e1484a67e6b0eaf08cc5e1d6301f8319 2024-11-14T14:41:05,267 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/e1484a67e6b0eaf08cc5e1d6301f8319 2024-11-14T14:41:05,268 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/e1484a67e6b0eaf08cc5e1d6301f8319 2024-11-14T14:41:05,268 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=25 2024-11-14T14:41:05,268 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure 6d725ecb42ad6f19c89d6ddaa229edc8, server=da1c6afcd1f9,40103,1731595261206 in 512 msec 2024-11-14T14:41:05,269 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1048): stopping wal replay for e1484a67e6b0eaf08cc5e1d6301f8319 2024-11-14T14:41:05,269 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1060): Cleaning up temporary data for e1484a67e6b0eaf08cc5e1d6301f8319 2024-11-14T14:41:05,270 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6d725ecb42ad6f19c89d6ddaa229edc8, ASSIGN in 795 msec 2024-11-14T14:41:05,271 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): writing seq id for e1484a67e6b0eaf08cc5e1d6301f8319 2024-11-14T14:41:05,274 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=22 2024-11-14T14:41:05,274 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure b2f6569935d4c7faa1470e22f0b79a20, server=da1c6afcd1f9,40103,1731595261206 in 503 msec 2024-11-14T14:41:05,275 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/e1484a67e6b0eaf08cc5e1d6301f8319/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T14:41:05,276 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1114): Opened e1484a67e6b0eaf08cc5e1d6301f8319; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63434587, jitterRate=-0.054750993847846985}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T14:41:05,276 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e1484a67e6b0eaf08cc5e1d6301f8319 2024-11-14T14:41:05,276 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1006): Region open journal for e1484a67e6b0eaf08cc5e1d6301f8319: Running coprocessor pre-open hook at 1731595265260Writing region info on filesystem at 1731595265260Initializing all the Stores at 1731595265262 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595265262Cleaning up temporary data from old regions at 1731595265269 (+7 ms)Running coprocessor post-open hooks at 1731595265276 (+7 ms)Region opened successfully at 1731595265276 2024-11-14T14:41:05,276 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=b2f6569935d4c7faa1470e22f0b79a20, ASSIGN in 801 msec 2024-11-14T14:41:05,277 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2236): Post open deploy tasks for test,qqq,1731595263719.e1484a67e6b0eaf08cc5e1d6301f8319., pid=49, masterSystemTime=1731595264906 2024-11-14T14:41:05,281 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2266): Finished post open deploy task for test,qqq,1731595263719.e1484a67e6b0eaf08cc5e1d6301f8319. 2024-11-14T14:41:05,281 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(153): Opened test,qqq,1731595263719.e1484a67e6b0eaf08cc5e1d6301f8319. 2024-11-14T14:41:05,282 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open test,ddd,1731595263719.8dbd58303faefb2f8b5972b0c3fc3aea. 2024-11-14T14:41:05,282 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => 8dbd58303faefb2f8b5972b0c3fc3aea, NAME => 'test,ddd,1731595263719.8dbd58303faefb2f8b5972b0c3fc3aea.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-11-14T14:41:05,282 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8dbd58303faefb2f8b5972b0c3fc3aea 2024-11-14T14:41:05,282 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated test,ddd,1731595263719.8dbd58303faefb2f8b5972b0c3fc3aea.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:41:05,283 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for 8dbd58303faefb2f8b5972b0c3fc3aea 2024-11-14T14:41:05,283 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for 8dbd58303faefb2f8b5972b0c3fc3aea 2024-11-14T14:41:05,284 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=29 2024-11-14T14:41:05,284 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 76be481b709a73fdf12e38c88e45d041, server=da1c6afcd1f9,46729,1731595261321 in 632 msec 2024-11-14T14:41:05,285 INFO [StoreOpener-8dbd58303faefb2f8b5972b0c3fc3aea-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8dbd58303faefb2f8b5972b0c3fc3aea 2024-11-14T14:41:05,287 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=76be481b709a73fdf12e38c88e45d041, ASSIGN in 812 msec 2024-11-14T14:41:05,287 INFO [StoreOpener-8dbd58303faefb2f8b5972b0c3fc3aea-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8dbd58303faefb2f8b5972b0c3fc3aea columnFamilyName family 2024-11-14T14:41:05,287 DEBUG [StoreOpener-8dbd58303faefb2f8b5972b0c3fc3aea-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:41:05,288 INFO [StoreOpener-8dbd58303faefb2f8b5972b0c3fc3aea-1 {}] regionserver.HStore(327): Store=8dbd58303faefb2f8b5972b0c3fc3aea/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:41:05,288 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for 8dbd58303faefb2f8b5972b0c3fc3aea 2024-11-14T14:41:05,289 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=e1484a67e6b0eaf08cc5e1d6301f8319, regionState=OPEN, openSeqNum=2, regionLocation=da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:05,290 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/8dbd58303faefb2f8b5972b0c3fc3aea 2024-11-14T14:41:05,291 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/8dbd58303faefb2f8b5972b0c3fc3aea 2024-11-14T14:41:05,293 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for 8dbd58303faefb2f8b5972b0c3fc3aea 2024-11-14T14:41:05,293 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for 8dbd58303faefb2f8b5972b0c3fc3aea 2024-11-14T14:41:05,293 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure e1484a67e6b0eaf08cc5e1d6301f8319, server=da1c6afcd1f9,40103,1731595261206 because future has completed 2024-11-14T14:41:05,295 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for 8dbd58303faefb2f8b5972b0c3fc3aea 2024-11-14T14:41:05,297 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=21 2024-11-14T14:41:05,297 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=21, state=SUCCESS, hasLock=false; OpenRegionProcedure e1484a67e6b0eaf08cc5e1d6301f8319, server=da1c6afcd1f9,40103,1731595261206 in 531 msec 2024-11-14T14:41:05,299 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/8dbd58303faefb2f8b5972b0c3fc3aea/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T14:41:05,300 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e1484a67e6b0eaf08cc5e1d6301f8319, ASSIGN in 825 msec 2024-11-14T14:41:05,300 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened 8dbd58303faefb2f8b5972b0c3fc3aea; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60288007, jitterRate=-0.10163868963718414}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T14:41:05,300 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8dbd58303faefb2f8b5972b0c3fc3aea 2024-11-14T14:41:05,300 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for 8dbd58303faefb2f8b5972b0c3fc3aea: Running coprocessor pre-open hook at 1731595265283Writing region info on filesystem at 1731595265283Initializing all the Stores at 1731595265284 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595265284Cleaning up temporary data from old regions at 1731595265293 (+9 ms)Running coprocessor post-open hooks at 1731595265300 (+7 ms)Region opened successfully at 1731595265300 2024-11-14T14:41:05,301 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ddd,1731595263719.8dbd58303faefb2f8b5972b0c3fc3aea., pid=55, masterSystemTime=1731595264906 2024-11-14T14:41:05,304 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ddd,1731595263719.8dbd58303faefb2f8b5972b0c3fc3aea. 2024-11-14T14:41:05,304 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened test,ddd,1731595263719.8dbd58303faefb2f8b5972b0c3fc3aea. 2024-11-14T14:41:05,304 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=8dbd58303faefb2f8b5972b0c3fc3aea, regionState=OPEN, openSeqNum=2, regionLocation=da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:05,307 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8dbd58303faefb2f8b5972b0c3fc3aea, server=da1c6afcd1f9,40103,1731595261206 because future has completed 2024-11-14T14:41:05,310 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=8 2024-11-14T14:41:05,310 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 8dbd58303faefb2f8b5972b0c3fc3aea, server=da1c6afcd1f9,40103,1731595261206 in 506 msec 2024-11-14T14:41:05,313 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=4 2024-11-14T14:41:05,313 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8dbd58303faefb2f8b5972b0c3fc3aea, ASSIGN in 837 msec 2024-11-14T14:41:05,314 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T14:41:05,314 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731595265314"}]},"ts":"1731595265314"} 2024-11-14T14:41:05,317 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLED in hbase:meta 2024-11-14T14:41:05,318 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T14:41:05,322 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test in 1.5910 sec 2024-11-14T14:41:05,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T14:41:05,902 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test completed 2024-11-14T14:41:05,902 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table test get assigned. Timeout = 60000ms 2024-11-14T14:41:05,904 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-14T14:41:05,959 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table test assigned to meta. Checking AM states. 2024-11-14T14:41:05,960 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-14T14:41:05,961 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table test assigned. 2024-11-14T14:41:05,970 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-14T14:41:05,979 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-14T14:41:06,027 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-14T14:41:06,049 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T14:41:06,081 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53178, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T14:41:06,101 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T14:41:06,128 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41872, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T14:41:06,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34057 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 3613f06851ef4465281b91596c2f4aa4 changed from -1.0 to 0.0, refreshing cache 2024-11-14T14:41:06,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34057 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 8dbd58303faefb2f8b5972b0c3fc3aea changed from -1.0 to 0.0, refreshing cache 2024-11-14T14:41:06,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34057 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region ad7ad94ee453836a534ae596990b42b8 changed from -1.0 to 0.0, refreshing cache 2024-11-14T14:41:06,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34057 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region d14b6f516265d1b90f14cf4519cf2b0f changed from -1.0 to 0.0, refreshing cache 2024-11-14T14:41:06,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34057 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f0bff83267f1352a4c903f322542cf14 changed from -1.0 to 0.0, refreshing cache 2024-11-14T14:41:06,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34057 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region e1484a67e6b0eaf08cc5e1d6301f8319 changed from -1.0 to 0.0, refreshing cache 2024-11-14T14:41:06,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34057 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region b2f6569935d4c7faa1470e22f0b79a20 changed from -1.0 to 0.0, refreshing cache 2024-11-14T14:41:06,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34057 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 6d725ecb42ad6f19c89d6ddaa229edc8 changed from -1.0 to 0.0, refreshing cache 2024-11-14T14:41:06,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34057 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 954bd925c25bfdb6e075a7471bbb1c4d changed from -1.0 to 0.0, refreshing cache 2024-11-14T14:41:06,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34057 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c1c1eeeedb7ad89130474f95f0b19f70 changed from -1.0 to 0.0, refreshing cache 2024-11-14T14:41:06,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34057 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 5e71d8bcd86c479813fc8c2e3a01b917 changed from -1.0 to 0.0, refreshing cache 2024-11-14T14:41:06,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34057 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 9cb318690a770d1645dfd64bec1a284b changed from -1.0 to 0.0, refreshing cache 2024-11-14T14:41:06,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34057 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1d7e464b03c5060f7367637b64f2eae0 changed from -1.0 to 0.0, refreshing cache 2024-11-14T14:41:06,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34057 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 9f8e87df66c884919328ba09109d819c changed from -1.0 to 0.0, refreshing cache 2024-11-14T14:41:06,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34057 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 624639c373b3612dcb71af916020ab1e changed from -1.0 to 0.0, refreshing cache 2024-11-14T14:41:06,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34057 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 71350ab931ae29ff2210bd9d6c3eeac5 changed from -1.0 to 0.0, refreshing cache 2024-11-14T14:41:06,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34057 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region dbc3f9fabaa1bd9fca0801fe6494782e changed from -1.0 to 0.0, refreshing cache 2024-11-14T14:41:06,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34057 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1316438481dbf11d64fb665f6cdd4eb6 changed from -1.0 to 0.0, refreshing cache 2024-11-14T14:41:06,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34057 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region b7030563781ca9d5e4d044b06d29712c changed from -1.0 to 0.0, refreshing cache 2024-11-14T14:41:06,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34057 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region d35b9113ecfdcde0da6d3c51d6891527 changed from -1.0 to 0.0, refreshing cache 2024-11-14T14:41:06,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34057 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 135a286d1aafce40676bb4d4370c9daf changed from -1.0 to 0.0, refreshing cache 2024-11-14T14:41:06,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34057 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 77f677c4c4cfcadb020d2436881cdeaa changed from -1.0 to 0.0, refreshing cache 2024-11-14T14:41:06,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34057 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 550a3973a29751310a867312ba0b03d1 changed from -1.0 to 0.0, refreshing cache 2024-11-14T14:41:06,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34057 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 177291d3bf2c8a0b1e9a3689929e9e50 changed from -1.0 to 0.0, refreshing cache 2024-11-14T14:41:06,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34057 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 2e2ab5762f8d8f31683b4f4994048f3e changed from -1.0 to 0.0, refreshing cache 2024-11-14T14:41:06,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34057 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 76be481b709a73fdf12e38c88e45d041 changed from -1.0 to 0.0, refreshing cache 2024-11-14T14:41:06,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34057 {}] rsgroup.RSGroupBasedLoadBalancer(138): Balancing RSGroup=default 2024-11-14T14:41:06,239 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34057 {}] rsgroup.RSGroupBasedLoadBalancer(151): Start Generate Balance plan for group: default 2024-11-14T14:41:06,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34057 {}] balancer.BaseLoadBalancer(619): Start Generate Balance plan for cluster. 2024-11-14T14:41:06,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34057 {}] balancer.BalancerClusterState(204): Hosts are {da1c6afcd1f9=0} racks are {/default-rack=0} 2024-11-14T14:41:06,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34057 {}] balancer.BalancerClusterState(303): server 0 has 9 regions 2024-11-14T14:41:06,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34057 {}] balancer.BalancerClusterState(303): server 1 has 9 regions 2024-11-14T14:41:06,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34057 {}] balancer.BalancerClusterState(303): server 2 has 9 regions 2024-11-14T14:41:06,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34057 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-14T14:41:06,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34057 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-14T14:41:06,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34057 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-14T14:41:06,241 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34057 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-14T14:41:06,241 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34057 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-14T14:41:06,241 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34057 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-14T14:41:06,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34057 {}] balancer.BalancerClusterState(326): Number of tables=2, number of hosts=1, number of racks=1 2024-11-14T14:41:06,247 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34057 {}] balancer.StochasticLoadBalancer(395): Cluster wide - skipping load balancing because weighted average imbalance=0.016474464579901153 <= threshold(0.025). If you want more aggressive balancing, either lower hbase.master.balancer.stochastic.minCostNeedBalance from 0.025 or increase the relative multiplier(s) of the specific cost function(s). functionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.0); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.0); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); 2024-11-14T14:41:06,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34057 {}] master.HMaster(2203): Balancer is going into sleep until next period in 300000ms 2024-11-14T14:41:06,251 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [900,000] milli-secs(wait.for.ratio=[1]) 2024-11-14T14:41:06,265 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34057 {}] master.MasterRpcServices(567): Client=jenkins//172.17.0.2 set balanceSwitch=false 2024-11-14T14:41:06,301 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.assignment.TestRollbackSCP#testFailAndRollback Thread=382, OpenFileDescriptor=641, MaxFileDescriptor=1048576, SystemLoadAverage=414, ProcessCount=11, AvailableMemoryMB=8233 2024-11-14T14:41:06,306 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table hbase:meta 2024-11-14T14:41:06,312 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=hbase:meta,,1 2024-11-14T14:41:06,313 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing da1c6afcd1f9,46729,1731595261321 2024-11-14T14:41:06,314 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server da1c6afcd1f9,46729,1731595261321: Simulated kill ***** 2024-11-14T14:41:06,314 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-14T14:41:06,317 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-14T14:41:06,320 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-14T14:41:06,324 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-14T14:41:06,324 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-14T14:41:06,325 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 305851872 }, "NonHeapMemoryUsage": { "committed": 151322624, "init": 7667712, "max": -1, "used": 149077488 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "da1c6afcd1f9" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "da1c6afcd1f9" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.Context": "regionserver", "tag.Hostname": "da1c6afcd1f9" } ] } 2024-11-14T14:41:06,332 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34057 {}] master.MasterRpcServices(700): da1c6afcd1f9,46729,1731595261321 reported a fatal error: ***** ABORTING region server da1c6afcd1f9,46729,1731595261321: Simulated kill ***** 2024-11-14T14:41:06,335 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'da1c6afcd1f9,46729,1731595261321' ***** 2024-11-14T14:41:06,335 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-11-14T14:41:06,336 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [15,000] milli-secs(wait.for.ratio=[1]) 2024-11-14T14:41:06,337 INFO [RS:1;da1c6afcd1f9:46729 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-14T14:41:06,337 INFO [RS:1;da1c6afcd1f9:46729 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-14T14:41:06,338 INFO [RS:1;da1c6afcd1f9:46729 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-14T14:41:06,338 INFO [RS:1;da1c6afcd1f9:46729 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T14:41:06,338 INFO [RS:1;da1c6afcd1f9:46729 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;da1c6afcd1f9:46729. 2024-11-14T14:41:06,338 DEBUG [RS:1;da1c6afcd1f9:46729 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T14:41:06,346 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-14T14:41:06,353 DEBUG [RS:1;da1c6afcd1f9:46729 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:41:06,364 INFO [regionserver/da1c6afcd1f9:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T14:41:06,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45021 is added to blk_1073741836_1012 (size=32114) 2024-11-14T14:41:06,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45825 is added to blk_1073741836_1012 (size=32114) 2024-11-14T14:41:06,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36849 is added to blk_1073741836_1012 (size=32114) 2024-11-14T14:41:06,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45021 is added to blk_1073741834_1010 (size=2237) 2024-11-14T14:41:06,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36849 is added to blk_1073741834_1010 (size=2237) 2024-11-14T14:41:06,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45825 is added to blk_1073741834_1010 (size=2237) 2024-11-14T14:41:06,404 DEBUG [RS:1;da1c6afcd1f9:46729 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:41:06,405 INFO [RS:1;da1c6afcd1f9:46729 {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T14:41:06,405 INFO [RS:1;da1c6afcd1f9:46729 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46729 2024-11-14T14:41:06,414 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34057-0x101a73a44750000, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T14:41:06,414 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46729-0x101a73a44750002, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/da1c6afcd1f9,46729,1731595261321 2024-11-14T14:41:06,414 INFO [RS:1;da1c6afcd1f9:46729 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T14:41:06,416 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [da1c6afcd1f9,46729,1731595261321] 2024-11-14T14:41:06,422 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/da1c6afcd1f9,46729,1731595261321 already deleted, retry=false 2024-11-14T14:41:06,422 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of da1c6afcd1f9,46729,1731595261321 on da1c6afcd1f9,34057,1731595260501 2024-11-14T14:41:06,429 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true 2024-11-14T14:41:06,435 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true 2024-11-14T14:41:06,435 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=57 for da1c6afcd1f9,46729,1731595261321 (carryingMeta=true) da1c6afcd1f9,46729,1731595261321/CRASHED/regionCount=9/lock=java.util.concurrent.locks.ReentrantReadWriteLock@3e3d292e[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-14T14:41:06,436 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true, isMeta: true 2024-11-14T14:41:06,438 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-14T14:41:06,439 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/WALs/da1c6afcd1f9,46729,1731595261321-splitting 2024-11-14T14:41:06,441 INFO [PEWorker-1 {}] master.SplitWALManager(105): da1c6afcd1f9,46729,1731595261321 WAL count=1, meta=true 2024-11-14T14:41:06,455 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta}] 2024-11-14T14:41:06,479 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:06,482 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363}] 2024-11-14T14:41:06,518 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46729-0x101a73a44750002, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T14:41:06,518 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46729-0x101a73a44750002, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T14:41:06,519 INFO [RS:1;da1c6afcd1f9:46729 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T14:41:06,520 INFO [RS:1;da1c6afcd1f9:46729 {}] regionserver.HRegionServer(1031): Exiting; stopping=da1c6afcd1f9,46729,1731595261321; zookeeper connection closed. 2024-11-14T14:41:06,520 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2040efbc {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2040efbc 2024-11-14T14:41:06,649 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46229 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=59 2024-11-14T14:41:06,661 INFO [regionserver/da1c6afcd1f9:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-14T14:41:06,662 INFO [regionserver/da1c6afcd1f9:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-14T14:41:06,722 INFO [RS_LOG_REPLAY_OPS-regionserver/da1c6afcd1f9:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(299): Splitting hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/WALs/da1c6afcd1f9,46729,1731595261321-splitting/da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, size=31.4 K (32114bytes) 2024-11-14T14:41:06,723 DEBUG [RS_LOG_REPLAY_OPS-regionserver/da1c6afcd1f9:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-14T14:41:06,723 DEBUG [RS_LOG_REPLAY_OPS-regionserver/da1c6afcd1f9:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-14T14:41:06,724 INFO [RS_LOG_REPLAY_OPS-regionserver/da1c6afcd1f9:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/WALs/da1c6afcd1f9,46729,1731595261321-splitting/da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta 2024-11-14T14:41:06,740 INFO [RS_LOG_REPLAY_OPS-regionserver/da1c6afcd1f9:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/WALs/da1c6afcd1f9,46729,1731595261321-splitting/da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta after 11ms 2024-11-14T14:41:06,751 INFO [RS_LOG_REPLAY_OPS-regionserver/da1c6afcd1f9:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(310): Open hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/WALs/da1c6afcd1f9,46729,1731595261321-splitting/da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta took 31ms 2024-11-14T14:41:06,780 DEBUG [RS_LOG_REPLAY_OPS-regionserver/da1c6afcd1f9:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-14T14:41:06,792 DEBUG [RS_LOG_REPLAY_OPS-regionserver/da1c6afcd1f9:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/WALs/da1c6afcd1f9,46729,1731595261321-splitting/da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta so closing down 2024-11-14T14:41:06,792 DEBUG [RS_LOG_REPLAY_OPS-regionserver/da1c6afcd1f9:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T14:41:06,799 INFO [RS_LOG_REPLAY_OPS-regionserver/da1c6afcd1f9:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta.temp 2024-11-14T14:41:06,813 INFO [RS_LOG_REPLAY_OPS-regionserver/da1c6afcd1f9:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta.temp 2024-11-14T14:41:06,818 INFO [RS_LOG_REPLAY_OPS-regionserver/da1c6afcd1f9:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T14:41:06,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45825 is added to blk_1073741864_1040 (size=31873) 2024-11-14T14:41:06,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36849 is added to blk_1073741864_1040 (size=31873) 2024-11-14T14:41:06,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45021 is added to blk_1073741864_1040 (size=31873) 2024-11-14T14:41:06,868 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta.temp (wrote 57 edits, skipped 0 edits in 3 ms) 2024-11-14T14:41:06,882 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta.temp to hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-14T14:41:06,889 INFO [RS_LOG_REPLAY_OPS-regionserver/da1c6afcd1f9:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(425): Processed 58 edits across 1 Regions in 120 ms; skipped=1; WAL=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/WALs/da1c6afcd1f9,46729,1731595261321-splitting/da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, size=31.4 K, length=32114, corrupted=false, cancelled=false 2024-11-14T14:41:06,889 DEBUG [RS_LOG_REPLAY_OPS-regionserver/da1c6afcd1f9:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(428): Completed split of hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/WALs/da1c6afcd1f9,46729,1731595261321-splitting/da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, journal: Splitting hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/WALs/da1c6afcd1f9,46729,1731595261321-splitting/da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, size=31.4 K (32114bytes) at 1731595266722Finishing writing output for hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/WALs/da1c6afcd1f9,46729,1731595261321-splitting/da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta so closing down at 1731595266792 (+70 ms)Creating recovered edits writer path=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta.temp at 1731595266813 (+21 ms)3 split writer threads finished at 1731595266818 (+5 ms)Closed recovered edits writer path=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta.temp (wrote 57 edits, skipped 0 edits in 3 ms) at 1731595266868 (+50 ms)Rename recovered edits hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta.temp to hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 at 1731595266882 (+14 ms)Processed 58 edits across 1 Regions in 120 ms; skipped=1; WAL=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/WALs/da1c6afcd1f9,46729,1731595261321-splitting/da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, size=31.4 K, length=32114, corrupted=false, cancelled=false at 1731595266889 (+7 ms) 2024-11-14T14:41:06,890 DEBUG [RS_LOG_REPLAY_OPS-regionserver/da1c6afcd1f9:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/WALs/da1c6afcd1f9,46729,1731595261321-splitting/da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta 2024-11-14T14:41:06,891 DEBUG [RS_LOG_REPLAY_OPS-regionserver/da1c6afcd1f9:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-14T14:41:06,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34057 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-14T14:41:06,906 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/WALs/da1c6afcd1f9,46729,1731595261321-splitting/da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta to hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/oldWALs 2024-11-14T14:41:06,911 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-14T14:41:06,911 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 in 424 msec 2024-11-14T14:41:06,913 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:06,920 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-14T14:41:06,920 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 in 470 msec 2024-11-14T14:41:06,923 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/WALs/da1c6afcd1f9,46729,1731595261321-splitting dir is empty, no logs to split. 2024-11-14T14:41:06,923 INFO [PEWorker-1 {}] master.SplitWALManager(105): da1c6afcd1f9,46729,1731595261321 WAL count=0, meta=true 2024-11-14T14:41:06,923 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if da1c6afcd1f9,46729,1731595261321 WAL splitting is done? wals=0, meta=true 2024-11-14T14:41:06,926 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-14T14:41:06,927 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T14:41:06,928 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-14T14:41:07,079 DEBUG [da1c6afcd1f9:34057 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-14T14:41:07,079 DEBUG [da1c6afcd1f9:34057 {}] balancer.BalancerClusterState(204): Hosts are {da1c6afcd1f9=0} racks are {/default-rack=0} 2024-11-14T14:41:07,079 DEBUG [da1c6afcd1f9:34057 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-14T14:41:07,079 DEBUG [da1c6afcd1f9:34057 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-14T14:41:07,079 DEBUG [da1c6afcd1f9:34057 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-14T14:41:07,080 DEBUG [da1c6afcd1f9:34057 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-14T14:41:07,080 INFO [da1c6afcd1f9:34057 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-14T14:41:07,080 INFO [da1c6afcd1f9:34057 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-14T14:41:07,080 DEBUG [da1c6afcd1f9:34057 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-14T14:41:07,080 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:07,083 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as da1c6afcd1f9,40103,1731595261206, state=OPENING 2024-11-14T14:41:07,086 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40103-0x101a73a44750001, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:07,086 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46229-0x101a73a44750003, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:07,086 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:07,086 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:07,086 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34057-0x101a73a44750000, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:07,087 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:07,088 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T14:41:07,089 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,40103,1731595261206}] 2024-11-14T14:41:07,250 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-14T14:41:07,250 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T14:41:07,250 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=61}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-14T14:41:07,255 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=da1c6afcd1f9%2C40103%2C1731595261206.meta, suffix=.meta, logDir=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/WALs/da1c6afcd1f9,40103,1731595261206, archiveDir=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/oldWALs, maxLogs=32 2024-11-14T14:41:07,274 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=61}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/WALs/da1c6afcd1f9,40103,1731595261206/da1c6afcd1f9%2C40103%2C1731595261206.meta.1731595267258.meta, exclude list is [], retry=0 2024-11-14T14:41:07,279 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36849,DS-0f3bbfa7-353f-42f1-ac96-37bfae4542a6,DISK] 2024-11-14T14:41:07,279 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45825,DS-d2124e34-cef4-477f-b806-d1dbb55af1e5,DISK] 2024-11-14T14:41:07,279 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45021,DS-728ae2cb-a461-4860-83b4-16a012c6b9ad,DISK] 2024-11-14T14:41:07,283 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/WALs/da1c6afcd1f9,40103,1731595261206/da1c6afcd1f9%2C40103%2C1731595261206.meta.1731595267258.meta 2024-11-14T14:41:07,284 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46259:46259),(127.0.0.1/127.0.0.1:44937:44937),(127.0.0.1/127.0.0.1:34623:34623)] 2024-11-14T14:41:07,284 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-14T14:41:07,285 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=61}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-14T14:41:07,285 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-14T14:41:07,285 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-14T14:41:07,285 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-14T14:41:07,285 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:41:07,286 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-14T14:41:07,286 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-14T14:41:07,293 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-14T14:41:07,294 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T14:41:07,295 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:41:07,295 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:41:07,296 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-14T14:41:07,297 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T14:41:07,297 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:41:07,297 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:41:07,298 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-14T14:41:07,299 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T14:41:07,299 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:41:07,299 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:41:07,300 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-14T14:41:07,301 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T14:41:07,301 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:41:07,301 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:41:07,302 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T14:41:07,302 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/hbase/meta/1588230740 2024-11-14T14:41:07,305 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/hbase/meta/1588230740 2024-11-14T14:41:07,307 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-14T14:41:07,333 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5793): Applied 264, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=60, path=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-14T14:41:07,339 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=28 KB heapSize=48.31 KB 2024-11-14T14:41:07,428 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/hbase/meta/1588230740/.tmp/info/0d3b8e1c199547a9890565db40701d75 is 138, key is test,bbb,1731595263719.b7030563781ca9d5e4d044b06d29712c./info:regioninfo/1731595265090/Put/seqid=0 2024-11-14T14:41:07,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45825 is added to blk_1073741866_1042 (size=34924) 2024-11-14T14:41:07,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45021 is added to blk_1073741866_1042 (size=34924) 2024-11-14T14:41:07,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36849 is added to blk_1073741866_1042 (size=34924) 2024-11-14T14:41:07,473 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.86 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/hbase/meta/1588230740/.tmp/info/0d3b8e1c199547a9890565db40701d75 2024-11-14T14:41:07,584 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/hbase/meta/1588230740/.tmp/ns/72d402e8161447c8bcc366c82097cbd9 is 43, key is default/ns:d/1731595263448/Put/seqid=0 2024-11-14T14:41:07,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45021 is added to blk_1073741867_1043 (size=5153) 2024-11-14T14:41:07,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45825 is added to blk_1073741867_1043 (size=5153) 2024-11-14T14:41:07,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36849 is added to blk_1073741867_1043 (size=5153) 2024-11-14T14:41:07,645 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/hbase/meta/1588230740/.tmp/ns/72d402e8161447c8bcc366c82097cbd9 2024-11-14T14:41:07,698 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/hbase/meta/1588230740/.tmp/table/cb5fbc3910d344c793a67f720f8e3fc2 is 40, key is test/table:state/1731595265314/Put/seqid=0 2024-11-14T14:41:07,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45021 is added to blk_1073741868_1044 (size=5165) 2024-11-14T14:41:07,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45825 is added to blk_1073741868_1044 (size=5165) 2024-11-14T14:41:07,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36849 is added to blk_1073741868_1044 (size=5165) 2024-11-14T14:41:07,825 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=72 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/hbase/meta/1588230740/.tmp/table/cb5fbc3910d344c793a67f720f8e3fc2 2024-11-14T14:41:07,839 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/hbase/meta/1588230740/.tmp/info/0d3b8e1c199547a9890565db40701d75 as hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/hbase/meta/1588230740/info/0d3b8e1c199547a9890565db40701d75 2024-11-14T14:41:07,852 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/hbase/meta/1588230740/info/0d3b8e1c199547a9890565db40701d75, entries=260, sequenceid=60, filesize=34.1 K 2024-11-14T14:41:07,866 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/hbase/meta/1588230740/.tmp/ns/72d402e8161447c8bcc366c82097cbd9 as hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/hbase/meta/1588230740/ns/72d402e8161447c8bcc366c82097cbd9 2024-11-14T14:41:07,878 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/hbase/meta/1588230740/ns/72d402e8161447c8bcc366c82097cbd9, entries=2, sequenceid=60, filesize=5.0 K 2024-11-14T14:41:07,879 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/hbase/meta/1588230740/.tmp/table/cb5fbc3910d344c793a67f720f8e3fc2 as hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/hbase/meta/1588230740/table/cb5fbc3910d344c793a67f720f8e3fc2 2024-11-14T14:41:07,890 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/hbase/meta/1588230740/table/cb5fbc3910d344c793a67f720f8e3fc2, entries=2, sequenceid=60, filesize=5.0 K 2024-11-14T14:41:07,896 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~28 KB/28672, heapSize ~48.02 KB/49168, currentSize=0 B/0 for 1588230740 in 556ms, sequenceid=60, compaction requested=false; wal=null 2024-11-14T14:41:07,898 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-14T14:41:07,911 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-14T14:41:07,913 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T14:41:07,913 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T14:41:07,914 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-14T14:41:07,916 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T14:41:07,947 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/hbase/meta/1588230740/recovered.edits/60.seqid, newMaxSeqId=60, maxSeqId=1 2024-11-14T14:41:07,949 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=61; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73581568, jitterRate=0.0964508056640625}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-14T14:41:07,949 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-14T14:41:07,949 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731595267286Writing region info on filesystem at 1731595267286Initializing all the Stores at 1731595267288 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731595267288Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731595267292 (+4 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595267292Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731595267292Obtaining lock to block concurrent updates at 1731595267339 (+47 ms)Preparing flush snapshotting stores in 1588230740 at 1731595267339Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=28672, getHeapSize=49408, getOffHeapSize=0, getCellsCount=264 at 1731595267343 (+4 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731595267343Flushing 1588230740/info: creating writer at 1731595267344 (+1 ms)Flushing 1588230740/info: appending metadata at 1731595267412 (+68 ms)Flushing 1588230740/info: closing flushed file at 1731595267415 (+3 ms)Flushing 1588230740/ns: creating writer at 1731595267542 (+127 ms)Flushing 1588230740/ns: appending metadata at 1731595267582 (+40 ms)Flushing 1588230740/ns: closing flushed file at 1731595267582Flushing 1588230740/table: creating writer at 1731595267657 (+75 ms)Flushing 1588230740/table: appending metadata at 1731595267696 (+39 ms)Flushing 1588230740/table: closing flushed file at 1731595267697 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@360fe848: reopening flushed file at 1731595267837 (+140 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@551b4b62: reopening flushed file at 1731595267865 (+28 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1868aa69: reopening flushed file at 1731595267878 (+13 ms)Finished flush of dataSize ~28 KB/28672, heapSize ~48.02 KB/49168, currentSize=0 B/0 for 1588230740 in 556ms, sequenceid=60, compaction requested=false; wal=null at 1731595267896 (+18 ms)Cleaning up temporary data from old regions at 1731595267913 (+17 ms)Running coprocessor post-open hooks at 1731595267949 (+36 ms)Region opened successfully at 1731595267949 2024-11-14T14:41:07,951 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=61, masterSystemTime=1731595267242 2024-11-14T14:41:07,957 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=61, regionLocation=da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:07,958 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as da1c6afcd1f9,40103,1731595261206, state=OPEN 2024-11-14T14:41:07,961 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40103-0x101a73a44750001, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:07,961 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46229-0x101a73a44750003, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:07,961 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34057-0x101a73a44750000, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:07,961 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:07,961 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:07,961 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:07,962 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=61, ppid=60, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:07,967 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-14T14:41:07,967 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,40103,1731595261206 in 874 msec 2024-11-14T14:41:07,971 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=57 2024-11-14T14:41:07,971 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 1.0420 sec 2024-11-14T14:41:07,972 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): da1c6afcd1f9,46729,1731595261321 had 9 regions 2024-11-14T14:41:07,974 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true, isMeta: false 2024-11-14T14:41:07,975 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-14T14:41:07,975 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-14T14:41:07,978 INFO [PEWorker-1 {}] master.SplitWALManager(105): da1c6afcd1f9,46729,1731595261321 WAL count=1, meta=false 2024-11-14T14:41:07,978 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836}] 2024-11-14T14:41:07,981 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:07,982 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363}] 2024-11-14T14:41:08,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46229 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=63 2024-11-14T14:41:08,161 INFO [RS_LOG_REPLAY_OPS-regionserver/da1c6afcd1f9:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(299): Splitting hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/WALs/da1c6afcd1f9,46729,1731595261321-splitting/da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, size=2.2 K (2237bytes) 2024-11-14T14:41:08,161 INFO [RS_LOG_REPLAY_OPS-regionserver/da1c6afcd1f9:0-1 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/WALs/da1c6afcd1f9,46729,1731595261321-splitting/da1c6afcd1f9%2C46729%2C1731595261321.1731595262836 2024-11-14T14:41:08,162 INFO [RS_LOG_REPLAY_OPS-regionserver/da1c6afcd1f9:0-1 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/WALs/da1c6afcd1f9,46729,1731595261321-splitting/da1c6afcd1f9%2C46729%2C1731595261321.1731595262836 after 1ms 2024-11-14T14:41:08,167 INFO [RS_LOG_REPLAY_OPS-regionserver/da1c6afcd1f9:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(310): Open hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/WALs/da1c6afcd1f9,46729,1731595261321-splitting/da1c6afcd1f9%2C46729%2C1731595261321.1731595262836 took 5ms 2024-11-14T14:41:08,175 DEBUG [RS_LOG_REPLAY_OPS-regionserver/da1c6afcd1f9:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for d35b9113ecfdcde0da6d3c51d6891527: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-14T14:41:08,179 DEBUG [RS_LOG_REPLAY_OPS-regionserver/da1c6afcd1f9:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 177291d3bf2c8a0b1e9a3689929e9e50: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-14T14:41:08,181 DEBUG [RS_LOG_REPLAY_OPS-regionserver/da1c6afcd1f9:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 550a3973a29751310a867312ba0b03d1: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-14T14:41:08,183 DEBUG [RS_LOG_REPLAY_OPS-regionserver/da1c6afcd1f9:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 135a286d1aafce40676bb4d4370c9daf: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-14T14:41:08,186 DEBUG [RS_LOG_REPLAY_OPS-regionserver/da1c6afcd1f9:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 77f677c4c4cfcadb020d2436881cdeaa: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-14T14:41:08,188 DEBUG [RS_LOG_REPLAY_OPS-regionserver/da1c6afcd1f9:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for b7030563781ca9d5e4d044b06d29712c: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-14T14:41:08,191 DEBUG [RS_LOG_REPLAY_OPS-regionserver/da1c6afcd1f9:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 76be481b709a73fdf12e38c88e45d041: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-14T14:41:08,193 DEBUG [RS_LOG_REPLAY_OPS-regionserver/da1c6afcd1f9:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 2e2ab5762f8d8f31683b4f4994048f3e: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-14T14:41:08,193 DEBUG [RS_LOG_REPLAY_OPS-regionserver/da1c6afcd1f9:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/WALs/da1c6afcd1f9,46729,1731595261321-splitting/da1c6afcd1f9%2C46729%2C1731595261321.1731595262836 so closing down 2024-11-14T14:41:08,193 DEBUG [RS_LOG_REPLAY_OPS-regionserver/da1c6afcd1f9:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T14:41:08,196 INFO [RS_LOG_REPLAY_OPS-regionserver/da1c6afcd1f9:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T14:41:08,196 INFO [RS_LOG_REPLAY_OPS-regionserver/da1c6afcd1f9:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(425): Processed 8 edits across 0 Regions in 24 ms; skipped=8; WAL=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/WALs/da1c6afcd1f9,46729,1731595261321-splitting/da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, size=2.2 K, length=2237, corrupted=false, cancelled=false 2024-11-14T14:41:08,196 DEBUG [RS_LOG_REPLAY_OPS-regionserver/da1c6afcd1f9:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(428): Completed split of hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/WALs/da1c6afcd1f9,46729,1731595261321-splitting/da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, journal: Splitting hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/WALs/da1c6afcd1f9,46729,1731595261321-splitting/da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, size=2.2 K (2237bytes) at 1731595268161Finishing writing output for hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/WALs/da1c6afcd1f9,46729,1731595261321-splitting/da1c6afcd1f9%2C46729%2C1731595261321.1731595262836 so closing down at 1731595268193 (+32 ms)3 split writer threads finished at 1731595268196 (+3 ms)Processed 8 edits across 0 Regions in 24 ms; skipped=8; WAL=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/WALs/da1c6afcd1f9,46729,1731595261321-splitting/da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, size=2.2 K, length=2237, corrupted=false, cancelled=false at 1731595268196 2024-11-14T14:41:08,196 DEBUG [RS_LOG_REPLAY_OPS-regionserver/da1c6afcd1f9:0-1 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/WALs/da1c6afcd1f9,46729,1731595261321-splitting/da1c6afcd1f9%2C46729%2C1731595261321.1731595262836 2024-11-14T14:41:08,196 DEBUG [RS_LOG_REPLAY_OPS-regionserver/da1c6afcd1f9:0-1 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-14T14:41:08,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34057 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-14T14:41:08,203 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/WALs/da1c6afcd1f9,46729,1731595261321-splitting/da1c6afcd1f9%2C46729%2C1731595261321.1731595262836 to hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/oldWALs 2024-11-14T14:41:08,207 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-14T14:41:08,207 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 in 221 msec 2024-11-14T14:41:08,211 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:08,214 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=57 2024-11-14T14:41:08,214 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 in 233 msec 2024-11-14T14:41:08,217 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/WALs/da1c6afcd1f9,46729,1731595261321-splitting dir is empty, no logs to split. 2024-11-14T14:41:08,217 INFO [PEWorker-1 {}] master.SplitWALManager(105): da1c6afcd1f9,46729,1731595261321 WAL count=0, meta=false 2024-11-14T14:41:08,217 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if da1c6afcd1f9,46729,1731595261321 WAL splitting is done? wals=0, meta=false 2024-11-14T14:41:08,222 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN}, {pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN}, {pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN}, {pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=76be481b709a73fdf12e38c88e45d041, ASSIGN}, {pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=135a286d1aafce40676bb4d4370c9daf, ASSIGN}, {pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2e2ab5762f8d8f31683b4f4994048f3e, ASSIGN}, {pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=177291d3bf2c8a0b1e9a3689929e9e50, ASSIGN}, {pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=550a3973a29751310a867312ba0b03d1, ASSIGN}] 2024-11-14T14:41:08,225 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=550a3973a29751310a867312ba0b03d1, ASSIGN 2024-11-14T14:41:08,226 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=550a3973a29751310a867312ba0b03d1, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-14T14:41:08,228 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=177291d3bf2c8a0b1e9a3689929e9e50, ASSIGN 2024-11-14T14:41:08,229 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=177291d3bf2c8a0b1e9a3689929e9e50, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-14T14:41:08,230 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2e2ab5762f8d8f31683b4f4994048f3e, ASSIGN 2024-11-14T14:41:08,231 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=2e2ab5762f8d8f31683b4f4994048f3e, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-14T14:41:08,232 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=135a286d1aafce40676bb4d4370c9daf, ASSIGN 2024-11-14T14:41:08,233 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=135a286d1aafce40676bb4d4370c9daf, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-14T14:41:08,234 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=76be481b709a73fdf12e38c88e45d041, ASSIGN 2024-11-14T14:41:08,240 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=76be481b709a73fdf12e38c88e45d041, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-14T14:41:08,244 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN 2024-11-14T14:41:08,245 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-14T14:41:08,246 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN 2024-11-14T14:41:08,247 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-14T14:41:08,252 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN 2024-11-14T14:41:08,253 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-14T14:41:08,377 DEBUG [da1c6afcd1f9:34057 {}] balancer.BalancerClusterState(204): Hosts are {da1c6afcd1f9=0} racks are {/default-rack=0} 2024-11-14T14:41:08,377 DEBUG [da1c6afcd1f9:34057 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-14T14:41:08,378 DEBUG [da1c6afcd1f9:34057 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-14T14:41:08,378 DEBUG [da1c6afcd1f9:34057 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-14T14:41:08,378 DEBUG [da1c6afcd1f9:34057 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-14T14:41:08,378 INFO [da1c6afcd1f9:34057 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-14T14:41:08,378 INFO [da1c6afcd1f9:34057 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-14T14:41:08,378 DEBUG [da1c6afcd1f9:34057 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-14T14:41:08,378 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=2e2ab5762f8d8f31683b4f4994048f3e, regionState=OPENING, regionLocation=da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:08,379 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=550a3973a29751310a867312ba0b03d1, regionState=OPENING, regionLocation=da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:08,380 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=135a286d1aafce40676bb4d4370c9daf, regionState=OPENING, regionLocation=da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:08,380 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server da1c6afcd1f9:46729 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: da1c6afcd1f9/172.17.0.2:46729 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:41:08,383 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=da1c6afcd1f9,46729,1731595261321, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=da1c6afcd1f9,46729,1731595261321, seqNum=-1, error=java.net.ConnectException: Call to address=da1c6afcd1f9:46729 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: da1c6afcd1f9/172.17.0.2:46729 2024-11-14T14:41:08,384 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=da1c6afcd1f9,46729,1731595261321, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-14T14:41:08,384 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=da1c6afcd1f9,46729,1731595261321, seqNum=-1 from cache 2024-11-14T14:41:08,385 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=da1c6afcd1f9,46729,1731595261321, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=da1c6afcd1f9:46729 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: da1c6afcd1f9/172.17.0.2:46729 2024-11-14T14:41:08,387 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=65 updating hbase:meta row=b7030563781ca9d5e4d044b06d29712c, regionState=OPENING, regionLocation=da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:08,387 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T14:41:08,387 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=da1c6afcd1f9,40103,1731595261206, seqNum=-1] 2024-11-14T14:41:08,388 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address da1c6afcd1f9:46729 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: da1c6afcd1f9/172.17.0.2:46729 2024-11-14T14:41:08,388 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T14:41:08,389 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=76be481b709a73fdf12e38c88e45d041, regionState=OPENING, regionLocation=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:08,390 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=177291d3bf2c8a0b1e9a3689929e9e50, regionState=OPENING, regionLocation=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:08,391 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=77f677c4c4cfcadb020d2436881cdeaa, regionState=OPENING, regionLocation=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:08,391 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56589, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T14:41:08,391 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=d35b9113ecfdcde0da6d3c51d6891527, regionState=OPENING, regionLocation=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:08,393 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server da1c6afcd1f9:46729 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: da1c6afcd1f9/172.17.0.2:46729 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:41:08,395 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=da1c6afcd1f9,46729,1731595261321, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=da1c6afcd1f9,40103,1731595261206, seqNum=-1, error=java.net.ConnectException: Call to address=da1c6afcd1f9:46729 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: da1c6afcd1f9/172.17.0.2:46729 2024-11-14T14:41:08,395 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address da1c6afcd1f9:46729 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: da1c6afcd1f9/172.17.0.2:46729 2024-11-14T14:41:08,398 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN because future has completed 2024-11-14T14:41:08,398 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7030563781ca9d5e4d044b06d29712c, server=da1c6afcd1f9,40103,1731595261206}] 2024-11-14T14:41:08,400 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=76be481b709a73fdf12e38c88e45d041, ASSIGN because future has completed 2024-11-14T14:41:08,401 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 76be481b709a73fdf12e38c88e45d041, server=da1c6afcd1f9,46229,1731595261363}] 2024-11-14T14:41:08,403 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=177291d3bf2c8a0b1e9a3689929e9e50, ASSIGN because future has completed 2024-11-14T14:41:08,404 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN because future has completed 2024-11-14T14:41:08,407 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN because future has completed 2024-11-14T14:41:08,408 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure d35b9113ecfdcde0da6d3c51d6891527, server=da1c6afcd1f9,46229,1731595261363}] 2024-11-14T14:41:08,413 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 77f677c4c4cfcadb020d2436881cdeaa, server=da1c6afcd1f9,46229,1731595261363}] 2024-11-14T14:41:08,419 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 177291d3bf2c8a0b1e9a3689929e9e50, server=da1c6afcd1f9,46229,1731595261363}] 2024-11-14T14:41:08,491 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=2e2ab5762f8d8f31683b4f4994048f3e, ASSIGN because future has completed 2024-11-14T14:41:08,491 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206}] 2024-11-14T14:41:08,493 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=550a3973a29751310a867312ba0b03d1, ASSIGN because future has completed 2024-11-14T14:41:08,496 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 550a3973a29751310a867312ba0b03d1, server=da1c6afcd1f9,40103,1731595261206}] 2024-11-14T14:41:08,500 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=135a286d1aafce40676bb4d4370c9daf, ASSIGN because future has completed 2024-11-14T14:41:08,503 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 135a286d1aafce40676bb4d4370c9daf, server=da1c6afcd1f9,40103,1731595261206}] 2024-11-14T14:41:08,562 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(132): Open test,yyy,1731595263719.76be481b709a73fdf12e38c88e45d041. 2024-11-14T14:41:08,563 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7752): Opening region: {ENCODED => 76be481b709a73fdf12e38c88e45d041, NAME => 'test,yyy,1731595263719.76be481b709a73fdf12e38c88e45d041.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-14T14:41:08,563 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 76be481b709a73fdf12e38c88e45d041 2024-11-14T14:41:08,563 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(898): Instantiated test,yyy,1731595263719.76be481b709a73fdf12e38c88e45d041.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:41:08,563 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7794): checking encryption for 76be481b709a73fdf12e38c88e45d041 2024-11-14T14:41:08,563 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7797): checking classloading for 76be481b709a73fdf12e38c88e45d041 2024-11-14T14:41:08,576 INFO [StoreOpener-76be481b709a73fdf12e38c88e45d041-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 76be481b709a73fdf12e38c88e45d041 2024-11-14T14:41:08,587 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(132): Open test,vvv,1731595263719.2e2ab5762f8d8f31683b4f4994048f3e. 2024-11-14T14:41:08,587 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7752): Opening region: {ENCODED => 2e2ab5762f8d8f31683b4f4994048f3e, NAME => 'test,vvv,1731595263719.2e2ab5762f8d8f31683b4f4994048f3e.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-14T14:41:08,588 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 2e2ab5762f8d8f31683b4f4994048f3e 2024-11-14T14:41:08,588 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(898): Instantiated test,vvv,1731595263719.2e2ab5762f8d8f31683b4f4994048f3e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:41:08,588 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7794): checking encryption for 2e2ab5762f8d8f31683b4f4994048f3e 2024-11-14T14:41:08,588 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7797): checking classloading for 2e2ab5762f8d8f31683b4f4994048f3e 2024-11-14T14:41:08,594 INFO [StoreOpener-2e2ab5762f8d8f31683b4f4994048f3e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 2e2ab5762f8d8f31683b4f4994048f3e 2024-11-14T14:41:08,596 INFO [StoreOpener-2e2ab5762f8d8f31683b4f4994048f3e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2e2ab5762f8d8f31683b4f4994048f3e columnFamilyName family 2024-11-14T14:41:08,596 DEBUG [StoreOpener-2e2ab5762f8d8f31683b4f4994048f3e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:41:08,597 INFO [StoreOpener-2e2ab5762f8d8f31683b4f4994048f3e-1 {}] regionserver.HStore(327): Store=2e2ab5762f8d8f31683b4f4994048f3e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:41:08,597 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1038): replaying wal for 2e2ab5762f8d8f31683b4f4994048f3e 2024-11-14T14:41:08,598 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/2e2ab5762f8d8f31683b4f4994048f3e 2024-11-14T14:41:08,602 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/2e2ab5762f8d8f31683b4f4994048f3e 2024-11-14T14:41:08,603 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1048): stopping wal replay for 2e2ab5762f8d8f31683b4f4994048f3e 2024-11-14T14:41:08,603 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1060): Cleaning up temporary data for 2e2ab5762f8d8f31683b4f4994048f3e 2024-11-14T14:41:08,606 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1093): writing seq id for 2e2ab5762f8d8f31683b4f4994048f3e 2024-11-14T14:41:08,606 INFO [StoreOpener-76be481b709a73fdf12e38c88e45d041-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 76be481b709a73fdf12e38c88e45d041 columnFamilyName family 2024-11-14T14:41:08,606 DEBUG [StoreOpener-76be481b709a73fdf12e38c88e45d041-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:41:08,607 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1114): Opened 2e2ab5762f8d8f31683b4f4994048f3e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60833302, jitterRate=-0.09351316094398499}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T14:41:08,607 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2e2ab5762f8d8f31683b4f4994048f3e 2024-11-14T14:41:08,607 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1006): Region open journal for 2e2ab5762f8d8f31683b4f4994048f3e: Running coprocessor pre-open hook at 1731595268588Writing region info on filesystem at 1731595268588Initializing all the Stores at 1731595268591 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595268591Cleaning up temporary data from old regions at 1731595268603 (+12 ms)Running coprocessor post-open hooks at 1731595268607 (+4 ms)Region opened successfully at 1731595268607 2024-11-14T14:41:08,609 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1731595263719.2e2ab5762f8d8f31683b4f4994048f3e., pid=77, masterSystemTime=1731595268576 2024-11-14T14:41:08,612 INFO [StoreOpener-76be481b709a73fdf12e38c88e45d041-1 {}] regionserver.HStore(327): Store=76be481b709a73fdf12e38c88e45d041/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:41:08,612 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1038): replaying wal for 76be481b709a73fdf12e38c88e45d041 2024-11-14T14:41:08,616 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/76be481b709a73fdf12e38c88e45d041 2024-11-14T14:41:08,619 WARN [PEWorker-1 {}] procedure2.ProcedureTestingUtility(201): Set Kill before store update in rollback to: true 2024-11-14T14:41:08,619 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:08,620 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1844): CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206 java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T14:41:08,623 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/76be481b709a73fdf12e38c88e45d041 2024-11-14T14:41:08,624 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1048): stopping wal replay for 76be481b709a73fdf12e38c88e45d041 2024-11-14T14:41:08,624 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1060): Cleaning up temporary data for 76be481b709a73fdf12e38c88e45d041 2024-11-14T14:41:08,621 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1731595263719.2e2ab5762f8d8f31683b4f4994048f3e. 2024-11-14T14:41:08,625 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(153): Opened test,vvv,1731595263719.2e2ab5762f8d8f31683b4f4994048f3e. 2024-11-14T14:41:08,625 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(132): Open test,iii,1731595263719.135a286d1aafce40676bb4d4370c9daf. 2024-11-14T14:41:08,625 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7752): Opening region: {ENCODED => 135a286d1aafce40676bb4d4370c9daf, NAME => 'test,iii,1731595263719.135a286d1aafce40676bb4d4370c9daf.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-14T14:41:08,626 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 135a286d1aafce40676bb4d4370c9daf 2024-11-14T14:41:08,626 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(898): Instantiated test,iii,1731595263719.135a286d1aafce40676bb4d4370c9daf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:41:08,626 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7794): checking encryption for 135a286d1aafce40676bb4d4370c9daf 2024-11-14T14:41:08,626 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7797): checking classloading for 135a286d1aafce40676bb4d4370c9daf 2024-11-14T14:41:08,629 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1093): writing seq id for 76be481b709a73fdf12e38c88e45d041 2024-11-14T14:41:08,630 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1114): Opened 76be481b709a73fdf12e38c88e45d041; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64580590, jitterRate=-0.03767421841621399}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T14:41:08,630 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 76be481b709a73fdf12e38c88e45d041 2024-11-14T14:41:08,631 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1006): Region open journal for 76be481b709a73fdf12e38c88e45d041: Running coprocessor pre-open hook at 1731595268563Writing region info on filesystem at 1731595268563Initializing all the Stores at 1731595268565 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595268565Cleaning up temporary data from old regions at 1731595268624 (+59 ms)Running coprocessor post-open hooks at 1731595268630 (+6 ms)Region opened successfully at 1731595268630 2024-11-14T14:41:08,637 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1731595263719.76be481b709a73fdf12e38c88e45d041., pid=73, masterSystemTime=1731595268554 2024-11-14T14:41:08,641 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1731595263719.76be481b709a73fdf12e38c88e45d041. 2024-11-14T14:41:08,641 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(153): Opened test,yyy,1731595263719.76be481b709a73fdf12e38c88e45d041. 2024-11-14T14:41:08,641 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(132): Open test,lll,1731595263719.77f677c4c4cfcadb020d2436881cdeaa. 2024-11-14T14:41:08,641 INFO [StoreOpener-135a286d1aafce40676bb4d4370c9daf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 135a286d1aafce40676bb4d4370c9daf 2024-11-14T14:41:08,641 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7752): Opening region: {ENCODED => 77f677c4c4cfcadb020d2436881cdeaa, NAME => 'test,lll,1731595263719.77f677c4c4cfcadb020d2436881cdeaa.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-14T14:41:08,642 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 77f677c4c4cfcadb020d2436881cdeaa 2024-11-14T14:41:08,642 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(898): Instantiated test,lll,1731595263719.77f677c4c4cfcadb020d2436881cdeaa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:41:08,642 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7794): checking encryption for 77f677c4c4cfcadb020d2436881cdeaa 2024-11-14T14:41:08,642 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7797): checking classloading for 77f677c4c4cfcadb020d2436881cdeaa 2024-11-14T14:41:08,648 INFO [StoreOpener-135a286d1aafce40676bb4d4370c9daf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 135a286d1aafce40676bb4d4370c9daf columnFamilyName family 2024-11-14T14:41:08,648 DEBUG [StoreOpener-135a286d1aafce40676bb4d4370c9daf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:41:08,649 INFO [StoreOpener-135a286d1aafce40676bb4d4370c9daf-1 {}] regionserver.HStore(327): Store=135a286d1aafce40676bb4d4370c9daf/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:41:08,649 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1038): replaying wal for 135a286d1aafce40676bb4d4370c9daf 2024-11-14T14:41:08,651 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/135a286d1aafce40676bb4d4370c9daf 2024-11-14T14:41:08,652 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/135a286d1aafce40676bb4d4370c9daf 2024-11-14T14:41:08,653 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1048): stopping wal replay for 135a286d1aafce40676bb4d4370c9daf 2024-11-14T14:41:08,653 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1060): Cleaning up temporary data for 135a286d1aafce40676bb4d4370c9daf 2024-11-14T14:41:08,655 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1093): writing seq id for 135a286d1aafce40676bb4d4370c9daf 2024-11-14T14:41:08,657 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1114): Opened 135a286d1aafce40676bb4d4370c9daf; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70727317, jitterRate=0.05391915142536163}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T14:41:08,657 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 135a286d1aafce40676bb4d4370c9daf 2024-11-14T14:41:08,657 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1006): Region open journal for 135a286d1aafce40676bb4d4370c9daf: Running coprocessor pre-open hook at 1731595268626Writing region info on filesystem at 1731595268626Initializing all the Stores at 1731595268627 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595268627Cleaning up temporary data from old regions at 1731595268653 (+26 ms)Running coprocessor post-open hooks at 1731595268657 (+4 ms)Region opened successfully at 1731595268657 2024-11-14T14:41:08,658 INFO [StoreOpener-77f677c4c4cfcadb020d2436881cdeaa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 77f677c4c4cfcadb020d2436881cdeaa 2024-11-14T14:41:08,658 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1731595263719.135a286d1aafce40676bb4d4370c9daf., pid=79, masterSystemTime=1731595268576 2024-11-14T14:41:08,660 INFO [StoreOpener-77f677c4c4cfcadb020d2436881cdeaa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 77f677c4c4cfcadb020d2436881cdeaa columnFamilyName family 2024-11-14T14:41:08,661 DEBUG [StoreOpener-77f677c4c4cfcadb020d2436881cdeaa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:41:08,661 INFO [StoreOpener-77f677c4c4cfcadb020d2436881cdeaa-1 {}] regionserver.HStore(327): Store=77f677c4c4cfcadb020d2436881cdeaa/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:41:08,662 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1038): replaying wal for 77f677c4c4cfcadb020d2436881cdeaa 2024-11-14T14:41:08,662 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2266): Finished post open deploy task for test,iii,1731595263719.135a286d1aafce40676bb4d4370c9daf. 2024-11-14T14:41:08,662 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(153): Opened test,iii,1731595263719.135a286d1aafce40676bb4d4370c9daf. 2024-11-14T14:41:08,662 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] handler.AssignRegionHandler(132): Open test,bbb,1731595263719.b7030563781ca9d5e4d044b06d29712c. 2024-11-14T14:41:08,662 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7752): Opening region: {ENCODED => b7030563781ca9d5e4d044b06d29712c, NAME => 'test,bbb,1731595263719.b7030563781ca9d5e4d044b06d29712c.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-11-14T14:41:08,663 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test b7030563781ca9d5e4d044b06d29712c 2024-11-14T14:41:08,663 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(898): Instantiated test,bbb,1731595263719.b7030563781ca9d5e4d044b06d29712c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:41:08,663 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7794): checking encryption for b7030563781ca9d5e4d044b06d29712c 2024-11-14T14:41:08,663 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7797): checking classloading for b7030563781ca9d5e4d044b06d29712c 2024-11-14T14:41:08,664 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/77f677c4c4cfcadb020d2436881cdeaa 2024-11-14T14:41:08,665 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/77f677c4c4cfcadb020d2436881cdeaa 2024-11-14T14:41:08,668 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1048): stopping wal replay for 77f677c4c4cfcadb020d2436881cdeaa 2024-11-14T14:41:08,668 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1060): Cleaning up temporary data for 77f677c4c4cfcadb020d2436881cdeaa 2024-11-14T14:41:08,673 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): writing seq id for 77f677c4c4cfcadb020d2436881cdeaa 2024-11-14T14:41:08,674 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1114): Opened 77f677c4c4cfcadb020d2436881cdeaa; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65177174, jitterRate=-0.02878442406654358}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T14:41:08,674 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 77f677c4c4cfcadb020d2436881cdeaa 2024-11-14T14:41:08,674 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1006): Region open journal for 77f677c4c4cfcadb020d2436881cdeaa: Running coprocessor pre-open hook at 1731595268642Writing region info on filesystem at 1731595268642Initializing all the Stores at 1731595268649 (+7 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595268649Cleaning up temporary data from old regions at 1731595268668 (+19 ms)Running coprocessor post-open hooks at 1731595268674 (+6 ms)Region opened successfully at 1731595268674 2024-11-14T14:41:08,677 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1731595263719.77f677c4c4cfcadb020d2436881cdeaa., pid=75, masterSystemTime=1731595268554 2024-11-14T14:41:08,679 INFO [StoreOpener-b7030563781ca9d5e4d044b06d29712c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region b7030563781ca9d5e4d044b06d29712c 2024-11-14T14:41:08,680 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1731595263719.77f677c4c4cfcadb020d2436881cdeaa. 2024-11-14T14:41:08,680 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(153): Opened test,lll,1731595263719.77f677c4c4cfcadb020d2436881cdeaa. 2024-11-14T14:41:08,680 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(132): Open test,sss,1731595263719.177291d3bf2c8a0b1e9a3689929e9e50. 2024-11-14T14:41:08,681 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7752): Opening region: {ENCODED => 177291d3bf2c8a0b1e9a3689929e9e50, NAME => 'test,sss,1731595263719.177291d3bf2c8a0b1e9a3689929e9e50.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-14T14:41:08,681 INFO [StoreOpener-b7030563781ca9d5e4d044b06d29712c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b7030563781ca9d5e4d044b06d29712c columnFamilyName family 2024-11-14T14:41:08,681 DEBUG [StoreOpener-b7030563781ca9d5e4d044b06d29712c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:41:08,681 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 177291d3bf2c8a0b1e9a3689929e9e50 2024-11-14T14:41:08,681 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(898): Instantiated test,sss,1731595263719.177291d3bf2c8a0b1e9a3689929e9e50.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:41:08,681 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7794): checking encryption for 177291d3bf2c8a0b1e9a3689929e9e50 2024-11-14T14:41:08,681 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7797): checking classloading for 177291d3bf2c8a0b1e9a3689929e9e50 2024-11-14T14:41:08,682 INFO [StoreOpener-b7030563781ca9d5e4d044b06d29712c-1 {}] regionserver.HStore(327): Store=b7030563781ca9d5e4d044b06d29712c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:41:08,682 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1038): replaying wal for b7030563781ca9d5e4d044b06d29712c 2024-11-14T14:41:08,683 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/b7030563781ca9d5e4d044b06d29712c 2024-11-14T14:41:08,684 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/b7030563781ca9d5e4d044b06d29712c 2024-11-14T14:41:08,663 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=true, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1845) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 4 more 2024-11-14T14:41:08,685 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1048): stopping wal replay for b7030563781ca9d5e4d044b06d29712c 2024-11-14T14:41:08,685 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1060): Cleaning up temporary data for b7030563781ca9d5e4d044b06d29712c 2024-11-14T14:41:08,686 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-14T14:41:08,686 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 135a286d1aafce40676bb4d4370c9daf, server=da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:08,686 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T14:41:08,687 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 135a286d1aafce40676bb4d4370c9daf, server=da1c6afcd1f9,40103,1731595261206 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T14:41:08,688 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1093): writing seq id for b7030563781ca9d5e4d044b06d29712c 2024-11-14T14:41:08,689 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1114): Opened b7030563781ca9d5e4d044b06d29712c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65430921, jitterRate=-0.025003299117088318}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T14:41:08,689 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b7030563781ca9d5e4d044b06d29712c 2024-11-14T14:41:08,689 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1006): Region open journal for b7030563781ca9d5e4d044b06d29712c: Running coprocessor pre-open hook at 1731595268663Writing region info on filesystem at 1731595268663Initializing all the Stores at 1731595268665 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595268665Cleaning up temporary data from old regions at 1731595268685 (+20 ms)Running coprocessor post-open hooks at 1731595268689 (+4 ms)Region opened successfully at 1731595268689 2024-11-14T14:41:08,692 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionServer(2236): Post open deploy tasks for test,bbb,1731595263719.b7030563781ca9d5e4d044b06d29712c., pid=72, masterSystemTime=1731595268576 2024-11-14T14:41:08,695 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionServer(2266): Finished post open deploy task for test,bbb,1731595263719.b7030563781ca9d5e4d044b06d29712c. 2024-11-14T14:41:08,695 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] handler.AssignRegionHandler(153): Opened test,bbb,1731595263719.b7030563781ca9d5e4d044b06d29712c. 2024-11-14T14:41:08,695 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(132): Open test,ooo,1731595263719.550a3973a29751310a867312ba0b03d1. 2024-11-14T14:41:08,695 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7752): Opening region: {ENCODED => 550a3973a29751310a867312ba0b03d1, NAME => 'test,ooo,1731595263719.550a3973a29751310a867312ba0b03d1.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-11-14T14:41:08,696 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 550a3973a29751310a867312ba0b03d1 2024-11-14T14:41:08,696 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(898): Instantiated test,ooo,1731595263719.550a3973a29751310a867312ba0b03d1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:41:08,696 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7794): checking encryption for 550a3973a29751310a867312ba0b03d1 2024-11-14T14:41:08,696 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7797): checking classloading for 550a3973a29751310a867312ba0b03d1 2024-11-14T14:41:08,697 INFO [StoreOpener-177291d3bf2c8a0b1e9a3689929e9e50-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 177291d3bf2c8a0b1e9a3689929e9e50 2024-11-14T14:41:08,714 INFO [StoreOpener-550a3973a29751310a867312ba0b03d1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 550a3973a29751310a867312ba0b03d1 2024-11-14T14:41:08,715 INFO [StoreOpener-177291d3bf2c8a0b1e9a3689929e9e50-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 177291d3bf2c8a0b1e9a3689929e9e50 columnFamilyName family 2024-11-14T14:41:08,715 DEBUG [StoreOpener-177291d3bf2c8a0b1e9a3689929e9e50-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:41:08,715 INFO [StoreOpener-550a3973a29751310a867312ba0b03d1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 550a3973a29751310a867312ba0b03d1 columnFamilyName family 2024-11-14T14:41:08,715 DEBUG [StoreOpener-550a3973a29751310a867312ba0b03d1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:41:08,716 INFO [StoreOpener-550a3973a29751310a867312ba0b03d1-1 {}] regionserver.HStore(327): Store=550a3973a29751310a867312ba0b03d1/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:41:08,716 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1038): replaying wal for 550a3973a29751310a867312ba0b03d1 2024-11-14T14:41:08,717 INFO [StoreOpener-177291d3bf2c8a0b1e9a3689929e9e50-1 {}] regionserver.HStore(327): Store=177291d3bf2c8a0b1e9a3689929e9e50/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:41:08,717 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1038): replaying wal for 177291d3bf2c8a0b1e9a3689929e9e50 2024-11-14T14:41:08,717 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/550a3973a29751310a867312ba0b03d1 2024-11-14T14:41:08,718 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/177291d3bf2c8a0b1e9a3689929e9e50 2024-11-14T14:41:08,719 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/550a3973a29751310a867312ba0b03d1 2024-11-14T14:41:08,719 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/177291d3bf2c8a0b1e9a3689929e9e50 2024-11-14T14:41:08,720 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1048): stopping wal replay for 550a3973a29751310a867312ba0b03d1 2024-11-14T14:41:08,720 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1048): stopping wal replay for 177291d3bf2c8a0b1e9a3689929e9e50 2024-11-14T14:41:08,720 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1060): Cleaning up temporary data for 550a3973a29751310a867312ba0b03d1 2024-11-14T14:41:08,720 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1060): Cleaning up temporary data for 177291d3bf2c8a0b1e9a3689929e9e50 2024-11-14T14:41:08,724 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1093): writing seq id for 177291d3bf2c8a0b1e9a3689929e9e50 2024-11-14T14:41:08,724 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1093): writing seq id for 550a3973a29751310a867312ba0b03d1 2024-11-14T14:41:08,726 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1114): Opened 177291d3bf2c8a0b1e9a3689929e9e50; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59993509, jitterRate=-0.10602705180644989}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T14:41:08,726 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 177291d3bf2c8a0b1e9a3689929e9e50 2024-11-14T14:41:08,726 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1006): Region open journal for 177291d3bf2c8a0b1e9a3689929e9e50: Running coprocessor pre-open hook at 1731595268682Writing region info on filesystem at 1731595268682Initializing all the Stores at 1731595268686 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595268686Cleaning up temporary data from old regions at 1731595268720 (+34 ms)Running coprocessor post-open hooks at 1731595268726 (+6 ms)Region opened successfully at 1731595268726 2024-11-14T14:41:08,727 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1114): Opened 550a3973a29751310a867312ba0b03d1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65425236, jitterRate=-0.025088012218475342}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T14:41:08,727 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 550a3973a29751310a867312ba0b03d1 2024-11-14T14:41:08,727 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1006): Region open journal for 550a3973a29751310a867312ba0b03d1: Running coprocessor pre-open hook at 1731595268696Writing region info on filesystem at 1731595268696Initializing all the Stores at 1731595268697 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595268697Cleaning up temporary data from old regions at 1731595268720 (+23 ms)Running coprocessor post-open hooks at 1731595268727 (+7 ms)Region opened successfully at 1731595268727 2024-11-14T14:41:08,729 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1731595263719.177291d3bf2c8a0b1e9a3689929e9e50., pid=76, masterSystemTime=1731595268554 2024-11-14T14:41:08,733 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ooo,1731595263719.550a3973a29751310a867312ba0b03d1., pid=78, masterSystemTime=1731595268576 2024-11-14T14:41:08,735 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1731595263719.177291d3bf2c8a0b1e9a3689929e9e50. 2024-11-14T14:41:08,735 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(153): Opened test,sss,1731595263719.177291d3bf2c8a0b1e9a3689929e9e50. 2024-11-14T14:41:08,735 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(132): Open test,fff,1731595263719.d35b9113ecfdcde0da6d3c51d6891527. 2024-11-14T14:41:08,735 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7752): Opening region: {ENCODED => d35b9113ecfdcde0da6d3c51d6891527, NAME => 'test,fff,1731595263719.d35b9113ecfdcde0da6d3c51d6891527.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-14T14:41:08,736 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d35b9113ecfdcde0da6d3c51d6891527 2024-11-14T14:41:08,736 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(898): Instantiated test,fff,1731595263719.d35b9113ecfdcde0da6d3c51d6891527.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:41:08,736 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7794): checking encryption for d35b9113ecfdcde0da6d3c51d6891527 2024-11-14T14:41:08,736 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7797): checking classloading for d35b9113ecfdcde0da6d3c51d6891527 2024-11-14T14:41:08,738 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ooo,1731595263719.550a3973a29751310a867312ba0b03d1. 2024-11-14T14:41:08,738 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(153): Opened test,ooo,1731595263719.550a3973a29751310a867312ba0b03d1. 2024-11-14T14:41:08,745 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-14T14:41:08,745 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T14:41:08,745 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T14:41:08,746 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T14:41:08,746 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-14T14:41:08,747 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-14T14:41:08,747 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-14T14:41:08,747 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T14:41:08,748 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-14T14:41:08,752 INFO [StoreOpener-d35b9113ecfdcde0da6d3c51d6891527-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d35b9113ecfdcde0da6d3c51d6891527 2024-11-14T14:41:08,755 INFO [StoreOpener-d35b9113ecfdcde0da6d3c51d6891527-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d35b9113ecfdcde0da6d3c51d6891527 columnFamilyName family 2024-11-14T14:41:08,755 DEBUG [StoreOpener-d35b9113ecfdcde0da6d3c51d6891527-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:41:08,756 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-14T14:41:08,756 INFO [StoreOpener-d35b9113ecfdcde0da6d3c51d6891527-1 {}] regionserver.HStore(327): Store=d35b9113ecfdcde0da6d3c51d6891527/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:41:08,756 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1038): replaying wal for d35b9113ecfdcde0da6d3c51d6891527 2024-11-14T14:41:08,757 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/d35b9113ecfdcde0da6d3c51d6891527 2024-11-14T14:41:08,759 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/d35b9113ecfdcde0da6d3c51d6891527 2024-11-14T14:41:08,760 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1048): stopping wal replay for d35b9113ecfdcde0da6d3c51d6891527 2024-11-14T14:41:08,760 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1060): Cleaning up temporary data for d35b9113ecfdcde0da6d3c51d6891527 2024-11-14T14:41:08,760 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:08,760 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T14:41:08,760 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:08,760 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T14:41:08,760 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:08,760 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T14:41:08,760 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:08,760 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T14:41:08,760 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:08,760 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T14:41:08,761 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:08,761 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T14:41:08,761 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:08,761 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T14:41:08,761 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:08,761 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T14:41:08,761 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T14:41:08,761 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T14:41:08,761 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:08,761 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T14:41:08,761 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T14:41:08,761 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T14:41:08,762 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1093): writing seq id for d35b9113ecfdcde0da6d3c51d6891527 2024-11-14T14:41:08,763 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1114): Opened d35b9113ecfdcde0da6d3c51d6891527; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68183883, jitterRate=0.016019031405448914}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T14:41:08,764 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d35b9113ecfdcde0da6d3c51d6891527 2024-11-14T14:41:08,764 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1006): Region open journal for d35b9113ecfdcde0da6d3c51d6891527: Running coprocessor pre-open hook at 1731595268736Writing region info on filesystem at 1731595268736Initializing all the Stores at 1731595268738 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595268738Cleaning up temporary data from old regions at 1731595268760 (+22 ms)Running coprocessor post-open hooks at 1731595268764 (+4 ms)Region opened successfully at 1731595268764 2024-11-14T14:41:08,765 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1731595263719.d35b9113ecfdcde0da6d3c51d6891527., pid=74, masterSystemTime=1731595268554 2024-11-14T14:41:08,765 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T14:41:08,767 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-14T14:41:08,767 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34057 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T14:41:08,773 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true 2024-11-14T14:41:08,773 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:08,773 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:08,773 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T14:41:08,773 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:08,773 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:08,773 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:08,773 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN 2024-11-14T14:41:08,773 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN 2024-11-14T14:41:08,773 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN 2024-11-14T14:41:08,773 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76be481b709a73fdf12e38c88e45d041, ASSIGN 2024-11-14T14:41:08,773 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=135a286d1aafce40676bb4d4370c9daf, ASSIGN 2024-11-14T14:41:08,774 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e2ab5762f8d8f31683b4f4994048f3e, ASSIGN 2024-11-14T14:41:08,774 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=177291d3bf2c8a0b1e9a3689929e9e50, ASSIGN 2024-11-14T14:41:08,774 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=550a3973a29751310a867312ba0b03d1, ASSIGN 2024-11-14T14:41:08,774 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7030563781ca9d5e4d044b06d29712c, server=da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:08,774 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 76be481b709a73fdf12e38c88e45d041, server=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:08,774 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure d35b9113ecfdcde0da6d3c51d6891527, server=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:08,774 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 77f677c4c4cfcadb020d2436881cdeaa, server=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:08,774 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 177291d3bf2c8a0b1e9a3689929e9e50, server=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:08,774 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:08,774 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 550a3973a29751310a867312ba0b03d1, server=da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:08,774 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 135a286d1aafce40676bb4d4370c9daf, server=da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:08,774 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:08,774 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:08,774 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:08,774 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:08,774 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,40103,1731595261206 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:08,774 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:08,774 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:08,774 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:08,774 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN 2024-11-14T14:41:08,774 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:08,774 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN 2024-11-14T14:41:08,775 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:08,775 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN 2024-11-14T14:41:08,775 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76be481b709a73fdf12e38c88e45d041, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:08,775 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76be481b709a73fdf12e38c88e45d041, ASSIGN 2024-11-14T14:41:08,775 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=135a286d1aafce40676bb4d4370c9daf, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:08,775 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=135a286d1aafce40676bb4d4370c9daf, ASSIGN 2024-11-14T14:41:08,775 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e2ab5762f8d8f31683b4f4994048f3e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:08,775 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e2ab5762f8d8f31683b4f4994048f3e, ASSIGN 2024-11-14T14:41:08,775 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=177291d3bf2c8a0b1e9a3689929e9e50, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:08,775 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=177291d3bf2c8a0b1e9a3689929e9e50, ASSIGN 2024-11-14T14:41:08,775 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=550a3973a29751310a867312ba0b03d1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:08,775 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=550a3973a29751310a867312ba0b03d1, ASSIGN 2024-11-14T14:41:08,775 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7030563781ca9d5e4d044b06d29712c, server=da1c6afcd1f9,40103,1731595261206 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:08,775 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 76be481b709a73fdf12e38c88e45d041, server=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:08,775 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure d35b9113ecfdcde0da6d3c51d6891527, server=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:08,775 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 77f677c4c4cfcadb020d2436881cdeaa, server=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:08,775 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 177291d3bf2c8a0b1e9a3689929e9e50, server=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:08,775 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:08,775 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 550a3973a29751310a867312ba0b03d1, server=da1c6afcd1f9,40103,1731595261206 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:08,775 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 135a286d1aafce40676bb4d4370c9daf, server=da1c6afcd1f9,40103,1731595261206 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:08,778 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 27 msec 2024-11-14T14:41:08,781 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=61 2024-11-14T14:41:08,781 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=da1c6afcd1f9,40103,1731595261206, table=hbase:meta, region=1588230740 2024-11-14T14:41:08,781 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as da1c6afcd1f9,40103,1731595261206, state=OPEN 2024-11-14T14:41:08,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46229-0x101a73a44750003, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:08,785 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34057-0x101a73a44750000, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:08,785 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:08,785 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:08,785 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40103-0x101a73a44750001, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:08,785 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:08,790 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN to state=OFFLINE, location=null, table=test, region=d35b9113ecfdcde0da6d3c51d6891527 to restore RIT 2024-11-14T14:41:08,790 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN to state=OFFLINE, location=null, table=test, region=b7030563781ca9d5e4d044b06d29712c to restore RIT 2024-11-14T14:41:08,790 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN to state=OFFLINE, location=null, table=test, region=77f677c4c4cfcadb020d2436881cdeaa to restore RIT 2024-11-14T14:41:08,790 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76be481b709a73fdf12e38c88e45d041, ASSIGN to state=OFFLINE, location=null, table=test, region=76be481b709a73fdf12e38c88e45d041 to restore RIT 2024-11-14T14:41:08,790 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=135a286d1aafce40676bb4d4370c9daf, ASSIGN to state=OFFLINE, location=null, table=test, region=135a286d1aafce40676bb4d4370c9daf to restore RIT 2024-11-14T14:41:08,790 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e2ab5762f8d8f31683b4f4994048f3e, ASSIGN to state=OFFLINE, location=null, table=test, region=2e2ab5762f8d8f31683b4f4994048f3e to restore RIT 2024-11-14T14:41:08,790 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=177291d3bf2c8a0b1e9a3689929e9e50, ASSIGN to state=OFFLINE, location=null, table=test, region=177291d3bf2c8a0b1e9a3689929e9e50 to restore RIT 2024-11-14T14:41:08,790 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=550a3973a29751310a867312ba0b03d1, ASSIGN to state=OFFLINE, location=null, table=test, region=550a3973a29751310a867312ba0b03d1 to restore RIT 2024-11-14T14:41:08,797 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "da1c6afcd1f9" port: 46229 start_code: 1731595261363 } transition { transition_code: OPENED region_info { region_id: 1731595263719 table_name { namespace: "default" qualifier: "test" } start_key: "fff" end_key: "ggg" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 74 initiating_master_active_time: 1731595261447 }; retry (#0) after 1000ms delay (Master is coming online...). org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-14T14:41:08,826 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731595298826 2024-11-14T14:41:08,826 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-14T14:41:08,839 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T14:41:08,842 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T14:41:08,852 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-14T14:41:08,854 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-14T14:41:08,854 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 550a3973a29751310a867312ba0b03d1, server=da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:08,854 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T14:41:08,854 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 550a3973a29751310a867312ba0b03d1, server=da1c6afcd1f9,40103,1731595261206 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T14:41:08,891 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3613f06851ef4465281b91596c2f4aa4, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:08,892 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7030563781ca9d5e4d044b06d29712c, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:08,892 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1c1eeeedb7ad89130474f95f0b19f70, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:08,893 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8dbd58303faefb2f8b5972b0c3fc3aea, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:08,893 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e71d8bcd86c479813fc8c2e3a01b917, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:08,893 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d35b9113ecfdcde0da6d3c51d6891527, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:08,894 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9cb318690a770d1645dfd64bec1a284b, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:08,894 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad7ad94ee453836a534ae596990b42b8, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:08,894 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=135a286d1aafce40676bb4d4370c9daf, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:08,895 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d14b6f516265d1b90f14cf4519cf2b0f, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:08,895 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d7e464b03c5060f7367637b64f2eae0, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:08,895 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77f677c4c4cfcadb020d2436881cdeaa, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:08,896 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9f8e87df66c884919328ba09109d819c, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:08,899 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f0bff83267f1352a4c903f322542cf14, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:08,899 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=550a3973a29751310a867312ba0b03d1, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:08,900 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=624639c373b3612dcb71af916020ab1e, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:08,900 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e1484a67e6b0eaf08cc5e1d6301f8319, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:08,901 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b2f6569935d4c7faa1470e22f0b79a20, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:08,901 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=177291d3bf2c8a0b1e9a3689929e9e50, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:08,901 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71350ab931ae29ff2210bd9d6c3eeac5, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:08,902 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d725ecb42ad6f19c89d6ddaa229edc8, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:08,902 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e2ab5762f8d8f31683b4f4994048f3e, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:08,902 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbc3f9fabaa1bd9fca0801fe6494782e, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:08,903 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=954bd925c25bfdb6e075a7471bbb1c4d, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:08,903 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76be481b709a73fdf12e38c88e45d041, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:08,903 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1316438481dbf11d64fb665f6cdd4eb6, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:08,904 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-14T14:41:08,904 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731595328904 2024-11-14T14:41:08,904 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731595388904 2024-11-14T14:41:08,904 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 77 msec 2024-11-14T14:41:08,904 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-14T14:41:08,904 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T14:41:08,904 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T14:41:08,904 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T14:41:08,905 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-14T14:41:08,905 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-14T14:41:08,905 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-14T14:41:08,905 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T14:41:08,906 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-14T14:41:08,914 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34057 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T14:41:08,920 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-14T14:41:08,921 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:08,921 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T14:41:08,921 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:08,921 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T14:41:08,921 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:08,921 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T14:41:08,921 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:08,921 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T14:41:08,921 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:08,921 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:08,921 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T14:41:08,921 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:08,921 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T14:41:08,921 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:08,921 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T14:41:08,921 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T14:41:08,921 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T14:41:08,921 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:08,921 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T14:41:08,921 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T14:41:08,921 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T14:41:08,922 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T14:41:08,923 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-14T14:41:08,931 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true 2024-11-14T14:41:08,931 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:08,931 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:08,931 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T14:41:08,931 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:08,931 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:08,931 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:08,931 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN 2024-11-14T14:41:08,931 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN 2024-11-14T14:41:08,931 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN 2024-11-14T14:41:08,931 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76be481b709a73fdf12e38c88e45d041, ASSIGN 2024-11-14T14:41:08,931 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=135a286d1aafce40676bb4d4370c9daf, ASSIGN 2024-11-14T14:41:08,931 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e2ab5762f8d8f31683b4f4994048f3e, ASSIGN 2024-11-14T14:41:08,931 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=177291d3bf2c8a0b1e9a3689929e9e50, ASSIGN 2024-11-14T14:41:08,931 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=550a3973a29751310a867312ba0b03d1, ASSIGN 2024-11-14T14:41:08,931 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7030563781ca9d5e4d044b06d29712c, server=da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:08,931 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 76be481b709a73fdf12e38c88e45d041, server=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:08,931 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure d35b9113ecfdcde0da6d3c51d6891527, server=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:08,931 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 77f677c4c4cfcadb020d2436881cdeaa, server=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:08,931 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 177291d3bf2c8a0b1e9a3689929e9e50, server=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:08,931 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:08,932 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 550a3973a29751310a867312ba0b03d1, server=da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:08,932 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:08,932 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:08,932 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:08,932 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:08,932 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,40103,1731595261206 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:08,932 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:08,932 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:08,932 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:08,932 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN 2024-11-14T14:41:08,932 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:08,932 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN 2024-11-14T14:41:08,932 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:08,932 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN 2024-11-14T14:41:08,932 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76be481b709a73fdf12e38c88e45d041, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:08,932 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76be481b709a73fdf12e38c88e45d041, ASSIGN 2024-11-14T14:41:08,932 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=135a286d1aafce40676bb4d4370c9daf, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:08,932 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=135a286d1aafce40676bb4d4370c9daf, ASSIGN 2024-11-14T14:41:08,932 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e2ab5762f8d8f31683b4f4994048f3e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:08,933 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e2ab5762f8d8f31683b4f4994048f3e, ASSIGN 2024-11-14T14:41:08,933 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=177291d3bf2c8a0b1e9a3689929e9e50, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:08,933 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=177291d3bf2c8a0b1e9a3689929e9e50, ASSIGN 2024-11-14T14:41:08,933 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=550a3973a29751310a867312ba0b03d1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:08,933 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=550a3973a29751310a867312ba0b03d1, ASSIGN 2024-11-14T14:41:08,933 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7030563781ca9d5e4d044b06d29712c, server=da1c6afcd1f9,40103,1731595261206 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:08,933 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 76be481b709a73fdf12e38c88e45d041, server=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:08,933 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure d35b9113ecfdcde0da6d3c51d6891527, server=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:08,933 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 77f677c4c4cfcadb020d2436881cdeaa, server=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:08,933 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 177291d3bf2c8a0b1e9a3689929e9e50, server=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:08,933 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:08,933 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 550a3973a29751310a867312ba0b03d1, server=da1c6afcd1f9,40103,1731595261206 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:08,933 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 26 msec 2024-11-14T14:41:08,960 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=61 2024-11-14T14:41:08,961 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=da1c6afcd1f9,40103,1731595261206, table=hbase:meta, region=1588230740 2024-11-14T14:41:08,961 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as da1c6afcd1f9,40103,1731595261206, state=OPEN 2024-11-14T14:41:08,963 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test' 2024-11-14T14:41:08,963 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40103-0x101a73a44750001, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:08,963 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46229-0x101a73a44750003, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:08,963 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:08,963 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:08,963 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34057-0x101a73a44750000, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:08,964 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:08,965 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN to state=OFFLINE, location=null, table=test, region=d35b9113ecfdcde0da6d3c51d6891527 to restore RIT 2024-11-14T14:41:08,965 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN to state=OFFLINE, location=null, table=test, region=b7030563781ca9d5e4d044b06d29712c to restore RIT 2024-11-14T14:41:08,965 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN to state=OFFLINE, location=null, table=test, region=77f677c4c4cfcadb020d2436881cdeaa to restore RIT 2024-11-14T14:41:08,965 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76be481b709a73fdf12e38c88e45d041, ASSIGN to state=OFFLINE, location=null, table=test, region=76be481b709a73fdf12e38c88e45d041 to restore RIT 2024-11-14T14:41:08,965 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=135a286d1aafce40676bb4d4370c9daf, ASSIGN to state=OFFLINE, location=null, table=test, region=135a286d1aafce40676bb4d4370c9daf to restore RIT 2024-11-14T14:41:08,965 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e2ab5762f8d8f31683b4f4994048f3e, ASSIGN to state=OFFLINE, location=null, table=test, region=2e2ab5762f8d8f31683b4f4994048f3e to restore RIT 2024-11-14T14:41:08,965 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=177291d3bf2c8a0b1e9a3689929e9e50, ASSIGN to state=OFFLINE, location=null, table=test, region=177291d3bf2c8a0b1e9a3689929e9e50 to restore RIT 2024-11-14T14:41:08,965 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=550a3973a29751310a867312ba0b03d1, ASSIGN to state=OFFLINE, location=null, table=test, region=550a3973a29751310a867312ba0b03d1 to restore RIT 2024-11-14T14:41:08,992 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731595298992 2024-11-14T14:41:08,992 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-14T14:41:08,992 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T14:41:08,994 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-14T14:41:08,998 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-14T14:41:08,998 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=77, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:08,998 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T14:41:08,999 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=77, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T14:41:09,006 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3613f06851ef4465281b91596c2f4aa4, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,006 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7030563781ca9d5e4d044b06d29712c, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,007 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1c1eeeedb7ad89130474f95f0b19f70, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,007 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8dbd58303faefb2f8b5972b0c3fc3aea, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,007 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e71d8bcd86c479813fc8c2e3a01b917, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,007 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d35b9113ecfdcde0da6d3c51d6891527, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,008 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9cb318690a770d1645dfd64bec1a284b, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,008 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad7ad94ee453836a534ae596990b42b8, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,008 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=135a286d1aafce40676bb4d4370c9daf, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,009 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d14b6f516265d1b90f14cf4519cf2b0f, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,009 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d7e464b03c5060f7367637b64f2eae0, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,009 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77f677c4c4cfcadb020d2436881cdeaa, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,009 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9f8e87df66c884919328ba09109d819c, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,010 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f0bff83267f1352a4c903f322542cf14, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,010 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=550a3973a29751310a867312ba0b03d1, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,010 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=624639c373b3612dcb71af916020ab1e, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,011 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e1484a67e6b0eaf08cc5e1d6301f8319, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,011 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b2f6569935d4c7faa1470e22f0b79a20, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,011 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=177291d3bf2c8a0b1e9a3689929e9e50, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,012 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71350ab931ae29ff2210bd9d6c3eeac5, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,012 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d725ecb42ad6f19c89d6ddaa229edc8, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,012 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e2ab5762f8d8f31683b4f4994048f3e, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,012 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbc3f9fabaa1bd9fca0801fe6494782e, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,013 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=954bd925c25bfdb6e075a7471bbb1c4d, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,013 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76be481b709a73fdf12e38c88e45d041, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,013 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1316438481dbf11d64fb665f6cdd4eb6, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,014 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-14T14:41:09,014 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731595329014 2024-11-14T14:41:09,014 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731595389014 2024-11-14T14:41:09,014 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 21 msec 2024-11-14T14:41:09,014 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-14T14:41:09,014 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T14:41:09,014 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T14:41:09,014 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T14:41:09,015 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-14T14:41:09,015 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-14T14:41:09,015 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-14T14:41:09,015 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T14:41:09,018 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 2 msec 2024-11-14T14:41:09,018 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34057 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T14:41:09,024 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-14T14:41:09,024 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,024 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T14:41:09,024 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,025 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T14:41:09,025 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,025 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T14:41:09,025 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,025 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T14:41:09,025 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,025 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,025 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T14:41:09,025 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,025 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T14:41:09,025 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,025 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T14:41:09,025 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T14:41:09,025 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,025 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T14:41:09,025 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T14:41:09,025 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T14:41:09,025 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T14:41:09,027 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-14T14:41:09,041 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true 2024-11-14T14:41:09,041 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:09,041 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:09,042 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T14:41:09,042 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:09,042 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:09,042 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:09,042 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN 2024-11-14T14:41:09,042 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN 2024-11-14T14:41:09,042 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN 2024-11-14T14:41:09,042 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76be481b709a73fdf12e38c88e45d041, ASSIGN 2024-11-14T14:41:09,042 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=135a286d1aafce40676bb4d4370c9daf, ASSIGN 2024-11-14T14:41:09,042 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e2ab5762f8d8f31683b4f4994048f3e, ASSIGN 2024-11-14T14:41:09,042 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=177291d3bf2c8a0b1e9a3689929e9e50, ASSIGN 2024-11-14T14:41:09,042 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=550a3973a29751310a867312ba0b03d1, ASSIGN 2024-11-14T14:41:09,042 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7030563781ca9d5e4d044b06d29712c, server=da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:09,042 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 76be481b709a73fdf12e38c88e45d041, server=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:09,042 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure d35b9113ecfdcde0da6d3c51d6891527, server=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:09,044 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 77f677c4c4cfcadb020d2436881cdeaa, server=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:09,044 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 177291d3bf2c8a0b1e9a3689929e9e50, server=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:09,044 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:09,044 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,047 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,047 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,047 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,047 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,40103,1731595261206 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,047 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,047 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,047 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,047 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN 2024-11-14T14:41:09,047 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,047 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN 2024-11-14T14:41:09,047 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,047 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN 2024-11-14T14:41:09,047 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76be481b709a73fdf12e38c88e45d041, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,047 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76be481b709a73fdf12e38c88e45d041, ASSIGN 2024-11-14T14:41:09,047 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=135a286d1aafce40676bb4d4370c9daf, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,047 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=135a286d1aafce40676bb4d4370c9daf, ASSIGN 2024-11-14T14:41:09,047 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e2ab5762f8d8f31683b4f4994048f3e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,048 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e2ab5762f8d8f31683b4f4994048f3e, ASSIGN 2024-11-14T14:41:09,048 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=177291d3bf2c8a0b1e9a3689929e9e50, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,048 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=177291d3bf2c8a0b1e9a3689929e9e50, ASSIGN 2024-11-14T14:41:09,048 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=550a3973a29751310a867312ba0b03d1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,048 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=550a3973a29751310a867312ba0b03d1, ASSIGN 2024-11-14T14:41:09,048 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7030563781ca9d5e4d044b06d29712c, server=da1c6afcd1f9,40103,1731595261206 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,048 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 76be481b709a73fdf12e38c88e45d041, server=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,048 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure d35b9113ecfdcde0da6d3c51d6891527, server=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,048 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 77f677c4c4cfcadb020d2436881cdeaa, server=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,048 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 177291d3bf2c8a0b1e9a3689929e9e50, server=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,048 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,049 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 29 msec 2024-11-14T14:41:09,054 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=61 2024-11-14T14:41:09,054 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=da1c6afcd1f9,40103,1731595261206, table=hbase:meta, region=1588230740 2024-11-14T14:41:09,054 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as da1c6afcd1f9,40103,1731595261206, state=OPEN 2024-11-14T14:41:09,056 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34057-0x101a73a44750000, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:09,056 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40103-0x101a73a44750001, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:09,056 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:09,056 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46229-0x101a73a44750003, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:09,056 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:09,057 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:09,060 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN to state=OFFLINE, location=null, table=test, region=d35b9113ecfdcde0da6d3c51d6891527 to restore RIT 2024-11-14T14:41:09,060 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN to state=OFFLINE, location=null, table=test, region=b7030563781ca9d5e4d044b06d29712c to restore RIT 2024-11-14T14:41:09,060 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN to state=OFFLINE, location=null, table=test, region=77f677c4c4cfcadb020d2436881cdeaa to restore RIT 2024-11-14T14:41:09,060 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76be481b709a73fdf12e38c88e45d041, ASSIGN to state=OFFLINE, location=null, table=test, region=76be481b709a73fdf12e38c88e45d041 to restore RIT 2024-11-14T14:41:09,060 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=135a286d1aafce40676bb4d4370c9daf, ASSIGN to state=OFFLINE, location=null, table=test, region=135a286d1aafce40676bb4d4370c9daf to restore RIT 2024-11-14T14:41:09,060 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e2ab5762f8d8f31683b4f4994048f3e, ASSIGN to state=OFFLINE, location=null, table=test, region=2e2ab5762f8d8f31683b4f4994048f3e to restore RIT 2024-11-14T14:41:09,060 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=177291d3bf2c8a0b1e9a3689929e9e50, ASSIGN to state=OFFLINE, location=null, table=test, region=177291d3bf2c8a0b1e9a3689929e9e50 to restore RIT 2024-11-14T14:41:09,060 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=550a3973a29751310a867312ba0b03d1, ASSIGN to state=OFFLINE, location=null, table=test, region=550a3973a29751310a867312ba0b03d1 to restore RIT 2024-11-14T14:41:09,066 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731595299066 2024-11-14T14:41:09,066 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-14T14:41:09,068 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T14:41:09,071 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-14T14:41:09,073 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-14T14:41:09,073 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 177291d3bf2c8a0b1e9a3689929e9e50, server=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:09,073 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T14:41:09,075 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 177291d3bf2c8a0b1e9a3689929e9e50, server=da1c6afcd1f9,46229,1731595261363 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T14:41:09,077 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3613f06851ef4465281b91596c2f4aa4, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,078 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7030563781ca9d5e4d044b06d29712c, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,078 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1c1eeeedb7ad89130474f95f0b19f70, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,078 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8dbd58303faefb2f8b5972b0c3fc3aea, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,078 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e71d8bcd86c479813fc8c2e3a01b917, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,079 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d35b9113ecfdcde0da6d3c51d6891527, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,079 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9cb318690a770d1645dfd64bec1a284b, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,079 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad7ad94ee453836a534ae596990b42b8, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,079 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=135a286d1aafce40676bb4d4370c9daf, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,080 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d14b6f516265d1b90f14cf4519cf2b0f, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,080 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d7e464b03c5060f7367637b64f2eae0, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,080 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77f677c4c4cfcadb020d2436881cdeaa, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,081 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9f8e87df66c884919328ba09109d819c, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,081 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f0bff83267f1352a4c903f322542cf14, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,081 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=550a3973a29751310a867312ba0b03d1, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,081 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=624639c373b3612dcb71af916020ab1e, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,082 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e1484a67e6b0eaf08cc5e1d6301f8319, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,082 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b2f6569935d4c7faa1470e22f0b79a20, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,082 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=177291d3bf2c8a0b1e9a3689929e9e50, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,082 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71350ab931ae29ff2210bd9d6c3eeac5, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d725ecb42ad6f19c89d6ddaa229edc8, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e2ab5762f8d8f31683b4f4994048f3e, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbc3f9fabaa1bd9fca0801fe6494782e, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,084 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=954bd925c25bfdb6e075a7471bbb1c4d, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,084 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76be481b709a73fdf12e38c88e45d041, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,084 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1316438481dbf11d64fb665f6cdd4eb6, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,084 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-14T14:41:09,084 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731595329084 2024-11-14T14:41:09,084 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731595389084 2024-11-14T14:41:09,084 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 17 msec 2024-11-14T14:41:09,084 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-14T14:41:09,085 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T14:41:09,085 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T14:41:09,085 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T14:41:09,086 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-14T14:41:09,086 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-14T14:41:09,086 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-14T14:41:09,086 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T14:41:09,091 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 4 msec 2024-11-14T14:41:09,096 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-14T14:41:09,096 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,096 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T14:41:09,096 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,096 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T14:41:09,096 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,096 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T14:41:09,096 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,097 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T14:41:09,097 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,097 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,097 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,097 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T14:41:09,097 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,097 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T14:41:09,097 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T14:41:09,097 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,097 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T14:41:09,097 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T14:41:09,097 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T14:41:09,097 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T14:41:09,100 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-14T14:41:09,103 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true 2024-11-14T14:41:09,104 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:09,104 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:09,104 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T14:41:09,104 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:09,104 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:09,104 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:09,104 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN 2024-11-14T14:41:09,104 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN 2024-11-14T14:41:09,105 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN 2024-11-14T14:41:09,105 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76be481b709a73fdf12e38c88e45d041, ASSIGN 2024-11-14T14:41:09,105 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=135a286d1aafce40676bb4d4370c9daf, ASSIGN 2024-11-14T14:41:09,105 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e2ab5762f8d8f31683b4f4994048f3e, ASSIGN 2024-11-14T14:41:09,105 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=177291d3bf2c8a0b1e9a3689929e9e50, ASSIGN 2024-11-14T14:41:09,105 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=550a3973a29751310a867312ba0b03d1, ASSIGN 2024-11-14T14:41:09,105 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7030563781ca9d5e4d044b06d29712c, server=da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:09,105 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 76be481b709a73fdf12e38c88e45d041, server=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:09,105 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure d35b9113ecfdcde0da6d3c51d6891527, server=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:09,105 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 77f677c4c4cfcadb020d2436881cdeaa, server=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:09,106 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 177291d3bf2c8a0b1e9a3689929e9e50, server=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:09,106 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,106 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,106 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,106 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,106 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,40103,1731595261206 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,106 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,106 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,106 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,106 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN 2024-11-14T14:41:09,106 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,106 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN 2024-11-14T14:41:09,106 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,106 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN 2024-11-14T14:41:09,106 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76be481b709a73fdf12e38c88e45d041, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,106 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76be481b709a73fdf12e38c88e45d041, ASSIGN 2024-11-14T14:41:09,106 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=135a286d1aafce40676bb4d4370c9daf, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,106 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=135a286d1aafce40676bb4d4370c9daf, ASSIGN 2024-11-14T14:41:09,106 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e2ab5762f8d8f31683b4f4994048f3e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,107 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e2ab5762f8d8f31683b4f4994048f3e, ASSIGN 2024-11-14T14:41:09,107 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=177291d3bf2c8a0b1e9a3689929e9e50, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,107 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=177291d3bf2c8a0b1e9a3689929e9e50, ASSIGN 2024-11-14T14:41:09,107 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=550a3973a29751310a867312ba0b03d1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,107 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=550a3973a29751310a867312ba0b03d1, ASSIGN 2024-11-14T14:41:09,107 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7030563781ca9d5e4d044b06d29712c, server=da1c6afcd1f9,40103,1731595261206 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,107 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 76be481b709a73fdf12e38c88e45d041, server=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,107 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure d35b9113ecfdcde0da6d3c51d6891527, server=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,107 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 77f677c4c4cfcadb020d2436881cdeaa, server=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,107 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 177291d3bf2c8a0b1e9a3689929e9e50, server=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,107 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 16 msec 2024-11-14T14:41:09,107 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34057 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T14:41:09,108 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=61 2024-11-14T14:41:09,109 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=da1c6afcd1f9,40103,1731595261206, table=hbase:meta, region=1588230740 2024-11-14T14:41:09,109 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as da1c6afcd1f9,40103,1731595261206, state=OPEN 2024-11-14T14:41:09,111 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34057-0x101a73a44750000, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:09,111 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40103-0x101a73a44750001, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:09,111 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:09,111 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46229-0x101a73a44750003, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:09,111 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:09,111 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:09,112 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN to state=OFFLINE, location=null, table=test, region=d35b9113ecfdcde0da6d3c51d6891527 to restore RIT 2024-11-14T14:41:09,112 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN to state=OFFLINE, location=null, table=test, region=b7030563781ca9d5e4d044b06d29712c to restore RIT 2024-11-14T14:41:09,112 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN to state=OFFLINE, location=null, table=test, region=77f677c4c4cfcadb020d2436881cdeaa to restore RIT 2024-11-14T14:41:09,112 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76be481b709a73fdf12e38c88e45d041, ASSIGN to state=OFFLINE, location=null, table=test, region=76be481b709a73fdf12e38c88e45d041 to restore RIT 2024-11-14T14:41:09,112 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=135a286d1aafce40676bb4d4370c9daf, ASSIGN to state=OFFLINE, location=null, table=test, region=135a286d1aafce40676bb4d4370c9daf to restore RIT 2024-11-14T14:41:09,112 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e2ab5762f8d8f31683b4f4994048f3e, ASSIGN to state=OFFLINE, location=null, table=test, region=2e2ab5762f8d8f31683b4f4994048f3e to restore RIT 2024-11-14T14:41:09,112 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=177291d3bf2c8a0b1e9a3689929e9e50, ASSIGN to state=OFFLINE, location=null, table=test, region=177291d3bf2c8a0b1e9a3689929e9e50 to restore RIT 2024-11-14T14:41:09,113 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=550a3973a29751310a867312ba0b03d1, ASSIGN to state=OFFLINE, location=null, table=test, region=550a3973a29751310a867312ba0b03d1 to restore RIT 2024-11-14T14:41:09,134 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731595299134 2024-11-14T14:41:09,134 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-14T14:41:09,136 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T14:41:09,138 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-14T14:41:09,145 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-14T14:41:09,145 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=75, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 77f677c4c4cfcadb020d2436881cdeaa, server=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:09,145 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T14:41:09,145 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3613f06851ef4465281b91596c2f4aa4, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,146 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7030563781ca9d5e4d044b06d29712c, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,146 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1c1eeeedb7ad89130474f95f0b19f70, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,146 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8dbd58303faefb2f8b5972b0c3fc3aea, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,146 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e71d8bcd86c479813fc8c2e3a01b917, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,147 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d35b9113ecfdcde0da6d3c51d6891527, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,147 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9cb318690a770d1645dfd64bec1a284b, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,145 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=75, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 77f677c4c4cfcadb020d2436881cdeaa, server=da1c6afcd1f9,46229,1731595261363 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T14:41:09,147 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad7ad94ee453836a534ae596990b42b8, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,147 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=135a286d1aafce40676bb4d4370c9daf, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,148 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d14b6f516265d1b90f14cf4519cf2b0f, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,148 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d7e464b03c5060f7367637b64f2eae0, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,148 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77f677c4c4cfcadb020d2436881cdeaa, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,148 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9f8e87df66c884919328ba09109d819c, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,149 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f0bff83267f1352a4c903f322542cf14, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,149 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=550a3973a29751310a867312ba0b03d1, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,149 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=624639c373b3612dcb71af916020ab1e, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e1484a67e6b0eaf08cc5e1d6301f8319, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b2f6569935d4c7faa1470e22f0b79a20, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=177291d3bf2c8a0b1e9a3689929e9e50, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71350ab931ae29ff2210bd9d6c3eeac5, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,151 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d725ecb42ad6f19c89d6ddaa229edc8, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,151 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e2ab5762f8d8f31683b4f4994048f3e, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,151 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbc3f9fabaa1bd9fca0801fe6494782e, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,151 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=954bd925c25bfdb6e075a7471bbb1c4d, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,151 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76be481b709a73fdf12e38c88e45d041, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,152 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1316438481dbf11d64fb665f6cdd4eb6, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,152 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-14T14:41:09,152 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731595329152 2024-11-14T14:41:09,152 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731595389152 2024-11-14T14:41:09,152 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 17 msec 2024-11-14T14:41:09,152 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-14T14:41:09,152 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T14:41:09,152 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T14:41:09,152 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T14:41:09,152 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-14T14:41:09,153 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-14T14:41:09,153 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-14T14:41:09,153 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T14:41:09,154 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-14T14:41:09,163 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-14T14:41:09,163 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,163 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T14:41:09,163 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,163 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T14:41:09,163 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,163 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T14:41:09,163 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,163 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T14:41:09,163 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,163 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,163 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,163 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,163 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T14:41:09,164 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T14:41:09,164 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,164 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T14:41:09,164 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T14:41:09,164 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T14:41:09,164 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T14:41:09,165 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-14T14:41:09,170 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true 2024-11-14T14:41:09,171 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:09,171 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:09,171 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T14:41:09,171 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:09,171 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:09,171 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:09,171 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN 2024-11-14T14:41:09,171 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN 2024-11-14T14:41:09,171 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN 2024-11-14T14:41:09,171 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76be481b709a73fdf12e38c88e45d041, ASSIGN 2024-11-14T14:41:09,171 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=135a286d1aafce40676bb4d4370c9daf, ASSIGN 2024-11-14T14:41:09,171 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e2ab5762f8d8f31683b4f4994048f3e, ASSIGN 2024-11-14T14:41:09,171 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=177291d3bf2c8a0b1e9a3689929e9e50, ASSIGN 2024-11-14T14:41:09,171 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=550a3973a29751310a867312ba0b03d1, ASSIGN 2024-11-14T14:41:09,171 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7030563781ca9d5e4d044b06d29712c, server=da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:09,171 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 76be481b709a73fdf12e38c88e45d041, server=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:09,171 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure d35b9113ecfdcde0da6d3c51d6891527, server=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:09,171 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 77f677c4c4cfcadb020d2436881cdeaa, server=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:09,171 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,171 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,171 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,171 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,171 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,40103,1731595261206 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,171 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,171 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,171 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,172 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN 2024-11-14T14:41:09,172 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,172 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN 2024-11-14T14:41:09,172 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,172 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN 2024-11-14T14:41:09,172 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76be481b709a73fdf12e38c88e45d041, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,172 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76be481b709a73fdf12e38c88e45d041, ASSIGN 2024-11-14T14:41:09,172 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=135a286d1aafce40676bb4d4370c9daf, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,172 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=135a286d1aafce40676bb4d4370c9daf, ASSIGN 2024-11-14T14:41:09,172 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e2ab5762f8d8f31683b4f4994048f3e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,172 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e2ab5762f8d8f31683b4f4994048f3e, ASSIGN 2024-11-14T14:41:09,177 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=177291d3bf2c8a0b1e9a3689929e9e50, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,177 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=177291d3bf2c8a0b1e9a3689929e9e50, ASSIGN 2024-11-14T14:41:09,177 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=550a3973a29751310a867312ba0b03d1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,177 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=550a3973a29751310a867312ba0b03d1, ASSIGN 2024-11-14T14:41:09,177 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7030563781ca9d5e4d044b06d29712c, server=da1c6afcd1f9,40103,1731595261206 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,177 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 76be481b709a73fdf12e38c88e45d041, server=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,177 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure d35b9113ecfdcde0da6d3c51d6891527, server=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,177 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 77f677c4c4cfcadb020d2436881cdeaa, server=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,177 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 23 msec 2024-11-14T14:41:09,189 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=61 2024-11-14T14:41:09,189 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=da1c6afcd1f9,40103,1731595261206, table=hbase:meta, region=1588230740 2024-11-14T14:41:09,189 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as da1c6afcd1f9,40103,1731595261206, state=OPEN 2024-11-14T14:41:09,192 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34057-0x101a73a44750000, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:09,192 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46229-0x101a73a44750003, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:09,192 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:09,192 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:09,192 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40103-0x101a73a44750001, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:09,193 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:09,194 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN to state=OFFLINE, location=null, table=test, region=d35b9113ecfdcde0da6d3c51d6891527 to restore RIT 2024-11-14T14:41:09,194 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN to state=OFFLINE, location=null, table=test, region=b7030563781ca9d5e4d044b06d29712c to restore RIT 2024-11-14T14:41:09,194 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN to state=OFFLINE, location=null, table=test, region=77f677c4c4cfcadb020d2436881cdeaa to restore RIT 2024-11-14T14:41:09,194 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76be481b709a73fdf12e38c88e45d041, ASSIGN to state=OFFLINE, location=null, table=test, region=76be481b709a73fdf12e38c88e45d041 to restore RIT 2024-11-14T14:41:09,194 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=135a286d1aafce40676bb4d4370c9daf, ASSIGN to state=OFFLINE, location=null, table=test, region=135a286d1aafce40676bb4d4370c9daf to restore RIT 2024-11-14T14:41:09,195 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e2ab5762f8d8f31683b4f4994048f3e, ASSIGN to state=OFFLINE, location=null, table=test, region=2e2ab5762f8d8f31683b4f4994048f3e to restore RIT 2024-11-14T14:41:09,195 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=177291d3bf2c8a0b1e9a3689929e9e50, ASSIGN to state=OFFLINE, location=null, table=test, region=177291d3bf2c8a0b1e9a3689929e9e50 to restore RIT 2024-11-14T14:41:09,195 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=550a3973a29751310a867312ba0b03d1, ASSIGN to state=OFFLINE, location=null, table=test, region=550a3973a29751310a867312ba0b03d1 to restore RIT 2024-11-14T14:41:09,208 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731595299208 2024-11-14T14:41:09,208 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-14T14:41:09,208 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T14:41:09,214 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-14T14:41:09,216 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-14T14:41:09,216 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure d35b9113ecfdcde0da6d3c51d6891527, server=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:09,216 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T14:41:09,217 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure d35b9113ecfdcde0da6d3c51d6891527, server=da1c6afcd1f9,46229,1731595261363 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T14:41:09,217 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3613f06851ef4465281b91596c2f4aa4, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,218 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7030563781ca9d5e4d044b06d29712c, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,218 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1c1eeeedb7ad89130474f95f0b19f70, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,218 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8dbd58303faefb2f8b5972b0c3fc3aea, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,219 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e71d8bcd86c479813fc8c2e3a01b917, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,219 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d35b9113ecfdcde0da6d3c51d6891527, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,219 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9cb318690a770d1645dfd64bec1a284b, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,219 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad7ad94ee453836a534ae596990b42b8, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=135a286d1aafce40676bb4d4370c9daf, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d14b6f516265d1b90f14cf4519cf2b0f, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d7e464b03c5060f7367637b64f2eae0, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77f677c4c4cfcadb020d2436881cdeaa, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9f8e87df66c884919328ba09109d819c, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f0bff83267f1352a4c903f322542cf14, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=550a3973a29751310a867312ba0b03d1, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=624639c373b3612dcb71af916020ab1e, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e1484a67e6b0eaf08cc5e1d6301f8319, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b2f6569935d4c7faa1470e22f0b79a20, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=177291d3bf2c8a0b1e9a3689929e9e50, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71350ab931ae29ff2210bd9d6c3eeac5, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d725ecb42ad6f19c89d6ddaa229edc8, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e2ab5762f8d8f31683b4f4994048f3e, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbc3f9fabaa1bd9fca0801fe6494782e, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=954bd925c25bfdb6e075a7471bbb1c4d, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76be481b709a73fdf12e38c88e45d041, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,225 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1316438481dbf11d64fb665f6cdd4eb6, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,225 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-14T14:41:09,225 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731595329225 2024-11-14T14:41:09,225 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731595389225 2024-11-14T14:41:09,225 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 16 msec 2024-11-14T14:41:09,225 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-14T14:41:09,225 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T14:41:09,225 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T14:41:09,225 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T14:41:09,226 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-14T14:41:09,226 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-14T14:41:09,226 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-14T14:41:09,226 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34057 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T14:41:09,226 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T14:41:09,229 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 2 msec 2024-11-14T14:41:09,232 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-14T14:41:09,232 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,232 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T14:41:09,232 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,232 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T14:41:09,232 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,232 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,232 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T14:41:09,232 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,232 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,233 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,233 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,233 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T14:41:09,233 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T14:41:09,233 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,233 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T14:41:09,233 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T14:41:09,233 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T14:41:09,233 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T14:41:09,234 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-14T14:41:09,237 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true 2024-11-14T14:41:09,237 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:09,237 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:09,237 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T14:41:09,237 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:09,237 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:09,237 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:09,237 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN 2024-11-14T14:41:09,237 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN 2024-11-14T14:41:09,237 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN 2024-11-14T14:41:09,237 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76be481b709a73fdf12e38c88e45d041, ASSIGN 2024-11-14T14:41:09,237 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=135a286d1aafce40676bb4d4370c9daf, ASSIGN 2024-11-14T14:41:09,237 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e2ab5762f8d8f31683b4f4994048f3e, ASSIGN 2024-11-14T14:41:09,237 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=177291d3bf2c8a0b1e9a3689929e9e50, ASSIGN 2024-11-14T14:41:09,237 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=550a3973a29751310a867312ba0b03d1, ASSIGN 2024-11-14T14:41:09,237 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7030563781ca9d5e4d044b06d29712c, server=da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:09,237 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 76be481b709a73fdf12e38c88e45d041, server=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:09,237 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure d35b9113ecfdcde0da6d3c51d6891527, server=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:09,237 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,248 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,248 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,248 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,248 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,40103,1731595261206 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,248 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,248 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,248 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,248 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN 2024-11-14T14:41:09,248 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,248 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN 2024-11-14T14:41:09,248 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,249 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN 2024-11-14T14:41:09,249 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76be481b709a73fdf12e38c88e45d041, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,249 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76be481b709a73fdf12e38c88e45d041, ASSIGN 2024-11-14T14:41:09,249 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=135a286d1aafce40676bb4d4370c9daf, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,249 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=135a286d1aafce40676bb4d4370c9daf, ASSIGN 2024-11-14T14:41:09,249 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e2ab5762f8d8f31683b4f4994048f3e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,249 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e2ab5762f8d8f31683b4f4994048f3e, ASSIGN 2024-11-14T14:41:09,249 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=177291d3bf2c8a0b1e9a3689929e9e50, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,249 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=177291d3bf2c8a0b1e9a3689929e9e50, ASSIGN 2024-11-14T14:41:09,249 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=550a3973a29751310a867312ba0b03d1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,249 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=550a3973a29751310a867312ba0b03d1, ASSIGN 2024-11-14T14:41:09,249 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7030563781ca9d5e4d044b06d29712c, server=da1c6afcd1f9,40103,1731595261206 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,249 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 76be481b709a73fdf12e38c88e45d041, server=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,249 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure d35b9113ecfdcde0da6d3c51d6891527, server=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,249 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 20 msec 2024-11-14T14:41:09,268 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=61 2024-11-14T14:41:09,268 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=da1c6afcd1f9,40103,1731595261206, table=hbase:meta, region=1588230740 2024-11-14T14:41:09,268 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as da1c6afcd1f9,40103,1731595261206, state=OPEN 2024-11-14T14:41:09,271 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40103-0x101a73a44750001, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:09,271 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46229-0x101a73a44750003, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:09,271 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:09,271 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:09,271 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34057-0x101a73a44750000, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:09,272 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN to state=OFFLINE, location=null, table=test, region=d35b9113ecfdcde0da6d3c51d6891527 to restore RIT 2024-11-14T14:41:09,272 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:09,272 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN to state=OFFLINE, location=null, table=test, region=b7030563781ca9d5e4d044b06d29712c to restore RIT 2024-11-14T14:41:09,272 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN to state=OFFLINE, location=null, table=test, region=77f677c4c4cfcadb020d2436881cdeaa to restore RIT 2024-11-14T14:41:09,272 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76be481b709a73fdf12e38c88e45d041, ASSIGN to state=OFFLINE, location=null, table=test, region=76be481b709a73fdf12e38c88e45d041 to restore RIT 2024-11-14T14:41:09,272 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=135a286d1aafce40676bb4d4370c9daf, ASSIGN to state=OFFLINE, location=null, table=test, region=135a286d1aafce40676bb4d4370c9daf to restore RIT 2024-11-14T14:41:09,272 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e2ab5762f8d8f31683b4f4994048f3e, ASSIGN to state=OFFLINE, location=null, table=test, region=2e2ab5762f8d8f31683b4f4994048f3e to restore RIT 2024-11-14T14:41:09,272 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=177291d3bf2c8a0b1e9a3689929e9e50, ASSIGN to state=OFFLINE, location=null, table=test, region=177291d3bf2c8a0b1e9a3689929e9e50 to restore RIT 2024-11-14T14:41:09,273 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=550a3973a29751310a867312ba0b03d1, ASSIGN to state=OFFLINE, location=null, table=test, region=550a3973a29751310a867312ba0b03d1 to restore RIT 2024-11-14T14:41:09,275 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731595299275 2024-11-14T14:41:09,275 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-14T14:41:09,278 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T14:41:09,281 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-14T14:41:09,283 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3613f06851ef4465281b91596c2f4aa4, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,283 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7030563781ca9d5e4d044b06d29712c, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,283 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1c1eeeedb7ad89130474f95f0b19f70, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,284 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8dbd58303faefb2f8b5972b0c3fc3aea, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,284 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e71d8bcd86c479813fc8c2e3a01b917, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,284 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d35b9113ecfdcde0da6d3c51d6891527, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,284 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9cb318690a770d1645dfd64bec1a284b, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,284 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad7ad94ee453836a534ae596990b42b8, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,285 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=135a286d1aafce40676bb4d4370c9daf, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,285 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d14b6f516265d1b90f14cf4519cf2b0f, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,285 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d7e464b03c5060f7367637b64f2eae0, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,285 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77f677c4c4cfcadb020d2436881cdeaa, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,285 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9f8e87df66c884919328ba09109d819c, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,286 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f0bff83267f1352a4c903f322542cf14, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,286 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=550a3973a29751310a867312ba0b03d1, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,286 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=624639c373b3612dcb71af916020ab1e, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,286 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e1484a67e6b0eaf08cc5e1d6301f8319, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,287 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b2f6569935d4c7faa1470e22f0b79a20, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,287 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=177291d3bf2c8a0b1e9a3689929e9e50, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,287 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71350ab931ae29ff2210bd9d6c3eeac5, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,287 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d725ecb42ad6f19c89d6ddaa229edc8, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,287 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e2ab5762f8d8f31683b4f4994048f3e, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,288 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbc3f9fabaa1bd9fca0801fe6494782e, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,288 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=954bd925c25bfdb6e075a7471bbb1c4d, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,288 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76be481b709a73fdf12e38c88e45d041, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,289 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1316438481dbf11d64fb665f6cdd4eb6, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,289 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-14T14:41:09,289 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731595329289 2024-11-14T14:41:09,289 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731595389289 2024-11-14T14:41:09,289 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 14 msec 2024-11-14T14:41:09,289 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-14T14:41:09,289 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=73, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 76be481b709a73fdf12e38c88e45d041, server=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:09,290 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T14:41:09,290 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=73, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 76be481b709a73fdf12e38c88e45d041, server=da1c6afcd1f9,46229,1731595261363 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T14:41:09,539 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-14T14:41:09,539 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T14:41:09,539 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T14:41:09,539 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T14:41:09,540 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-14T14:41:09,540 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-14T14:41:09,540 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-14T14:41:09,540 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T14:41:09,541 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-14T14:41:09,544 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-14T14:41:09,544 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,544 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,544 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T14:41:09,545 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,545 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,545 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T14:41:09,545 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,545 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,545 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,545 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,545 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T14:41:09,545 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T14:41:09,545 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,545 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T14:41:09,545 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T14:41:09,545 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T14:41:09,545 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T14:41:09,547 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-14T14:41:09,549 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true 2024-11-14T14:41:09,549 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:09,549 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:09,549 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T14:41:09,549 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:09,550 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:09,550 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:09,550 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN 2024-11-14T14:41:09,550 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN 2024-11-14T14:41:09,550 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN 2024-11-14T14:41:09,550 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76be481b709a73fdf12e38c88e45d041, ASSIGN 2024-11-14T14:41:09,550 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=135a286d1aafce40676bb4d4370c9daf, ASSIGN 2024-11-14T14:41:09,550 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e2ab5762f8d8f31683b4f4994048f3e, ASSIGN 2024-11-14T14:41:09,550 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=177291d3bf2c8a0b1e9a3689929e9e50, ASSIGN 2024-11-14T14:41:09,550 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=550a3973a29751310a867312ba0b03d1, ASSIGN 2024-11-14T14:41:09,550 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7030563781ca9d5e4d044b06d29712c, server=da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:09,550 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 76be481b709a73fdf12e38c88e45d041, server=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:09,550 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,550 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,550 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,550 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,550 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,40103,1731595261206 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,550 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,550 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,550 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,550 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN 2024-11-14T14:41:09,550 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,550 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN 2024-11-14T14:41:09,551 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,551 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN 2024-11-14T14:41:09,551 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76be481b709a73fdf12e38c88e45d041, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,552 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76be481b709a73fdf12e38c88e45d041, ASSIGN 2024-11-14T14:41:09,552 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=135a286d1aafce40676bb4d4370c9daf, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,552 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=135a286d1aafce40676bb4d4370c9daf, ASSIGN 2024-11-14T14:41:09,552 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e2ab5762f8d8f31683b4f4994048f3e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,552 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e2ab5762f8d8f31683b4f4994048f3e, ASSIGN 2024-11-14T14:41:09,552 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=177291d3bf2c8a0b1e9a3689929e9e50, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,552 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=177291d3bf2c8a0b1e9a3689929e9e50, ASSIGN 2024-11-14T14:41:09,552 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=550a3973a29751310a867312ba0b03d1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,552 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=550a3973a29751310a867312ba0b03d1, ASSIGN 2024-11-14T14:41:09,552 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7030563781ca9d5e4d044b06d29712c, server=da1c6afcd1f9,40103,1731595261206 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,552 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 76be481b709a73fdf12e38c88e45d041, server=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,553 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 11 msec 2024-11-14T14:41:09,565 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=61 2024-11-14T14:41:09,565 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=da1c6afcd1f9,40103,1731595261206, table=hbase:meta, region=1588230740 2024-11-14T14:41:09,565 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as da1c6afcd1f9,40103,1731595261206, state=OPEN 2024-11-14T14:41:09,567 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46229-0x101a73a44750003, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:09,567 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40103-0x101a73a44750001, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:09,567 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34057-0x101a73a44750000, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:09,567 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:09,567 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:09,568 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:09,570 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN to state=OFFLINE, location=null, table=test, region=d35b9113ecfdcde0da6d3c51d6891527 to restore RIT 2024-11-14T14:41:09,571 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN to state=OFFLINE, location=null, table=test, region=b7030563781ca9d5e4d044b06d29712c to restore RIT 2024-11-14T14:41:09,571 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN to state=OFFLINE, location=null, table=test, region=77f677c4c4cfcadb020d2436881cdeaa to restore RIT 2024-11-14T14:41:09,571 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76be481b709a73fdf12e38c88e45d041, ASSIGN to state=OFFLINE, location=null, table=test, region=76be481b709a73fdf12e38c88e45d041 to restore RIT 2024-11-14T14:41:09,571 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=135a286d1aafce40676bb4d4370c9daf, ASSIGN to state=OFFLINE, location=null, table=test, region=135a286d1aafce40676bb4d4370c9daf to restore RIT 2024-11-14T14:41:09,571 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e2ab5762f8d8f31683b4f4994048f3e, ASSIGN to state=OFFLINE, location=null, table=test, region=2e2ab5762f8d8f31683b4f4994048f3e to restore RIT 2024-11-14T14:41:09,571 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=177291d3bf2c8a0b1e9a3689929e9e50, ASSIGN to state=OFFLINE, location=null, table=test, region=177291d3bf2c8a0b1e9a3689929e9e50 to restore RIT 2024-11-14T14:41:09,571 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=550a3973a29751310a867312ba0b03d1, ASSIGN to state=OFFLINE, location=null, table=test, region=550a3973a29751310a867312ba0b03d1 to restore RIT 2024-11-14T14:41:09,583 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731595299583 2024-11-14T14:41:09,583 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-14T14:41:09,591 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3613f06851ef4465281b91596c2f4aa4, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,591 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7030563781ca9d5e4d044b06d29712c, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,591 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1c1eeeedb7ad89130474f95f0b19f70, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,591 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8dbd58303faefb2f8b5972b0c3fc3aea, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,592 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e71d8bcd86c479813fc8c2e3a01b917, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,592 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d35b9113ecfdcde0da6d3c51d6891527, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,592 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9cb318690a770d1645dfd64bec1a284b, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,592 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad7ad94ee453836a534ae596990b42b8, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,593 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=135a286d1aafce40676bb4d4370c9daf, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,593 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d14b6f516265d1b90f14cf4519cf2b0f, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,593 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d7e464b03c5060f7367637b64f2eae0, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,593 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77f677c4c4cfcadb020d2436881cdeaa, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,594 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9f8e87df66c884919328ba09109d819c, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,583 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T14:41:09,594 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f0bff83267f1352a4c903f322542cf14, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,594 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=550a3973a29751310a867312ba0b03d1, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,594 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-14T14:41:09,594 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=624639c373b3612dcb71af916020ab1e, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,595 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e1484a67e6b0eaf08cc5e1d6301f8319, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,595 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b2f6569935d4c7faa1470e22f0b79a20, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,595 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=177291d3bf2c8a0b1e9a3689929e9e50, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,595 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71350ab931ae29ff2210bd9d6c3eeac5, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,596 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d725ecb42ad6f19c89d6ddaa229edc8, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,596 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e2ab5762f8d8f31683b4f4994048f3e, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,596 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbc3f9fabaa1bd9fca0801fe6494782e, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,596 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=954bd925c25bfdb6e075a7471bbb1c4d, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,597 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76be481b709a73fdf12e38c88e45d041, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,597 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1316438481dbf11d64fb665f6cdd4eb6, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,597 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-14T14:41:09,597 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731595329597 2024-11-14T14:41:09,597 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731595389597 2024-11-14T14:41:09,597 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 14 msec 2024-11-14T14:41:09,599 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-14T14:41:09,599 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7030563781ca9d5e4d044b06d29712c, server=da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:09,599 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T14:41:09,599 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7030563781ca9d5e4d044b06d29712c, server=da1c6afcd1f9,40103,1731595261206 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T14:41:09,800 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34057 {}] assignment.TransitRegionStateProcedure(535): There is no outstanding remote region procedure for state=OPENING, location=da1c6afcd1f9,46229,1731595261363, table=test, region=d35b9113ecfdcde0da6d3c51d6891527, serverName=da1c6afcd1f9,46229,1731595261363, code=OPENED, seqId=2, proc=pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN, should be a retry, ignore 2024-11-14T14:41:09,801 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2353): TRANSITION REPORTED server { host_name: "da1c6afcd1f9" port: 46229 start_code: 1731595261363 } transition { transition_code: OPENED region_info { region_id: 1731595263719 table_name { namespace: "default" qualifier: "test" } start_key: "fff" end_key: "ggg" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 74 initiating_master_active_time: 1731595261447 } 2024-11-14T14:41:09,801 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1731595263719.d35b9113ecfdcde0da6d3c51d6891527. 2024-11-14T14:41:09,801 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(153): Opened test,fff,1731595263719.d35b9113ecfdcde0da6d3c51d6891527. 2024-11-14T14:41:09,847 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-14T14:41:09,848 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T14:41:09,848 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T14:41:09,849 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T14:41:09,849 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-14T14:41:09,849 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-14T14:41:09,849 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-14T14:41:09,849 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34057 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T14:41:09,850 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T14:41:09,851 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-14T14:41:09,853 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-14T14:41:09,853 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,853 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,853 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T14:41:09,853 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,853 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,853 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,853 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,853 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,853 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,853 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T14:41:09,853 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T14:41:09,853 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,853 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T14:41:09,853 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T14:41:09,853 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T14:41:09,854 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T14:41:09,855 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-14T14:41:09,858 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true 2024-11-14T14:41:09,858 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:09,858 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:09,858 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T14:41:09,858 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:09,858 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:09,858 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:09,858 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN 2024-11-14T14:41:09,858 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN 2024-11-14T14:41:09,858 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN 2024-11-14T14:41:09,858 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76be481b709a73fdf12e38c88e45d041, ASSIGN 2024-11-14T14:41:09,858 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=135a286d1aafce40676bb4d4370c9daf, ASSIGN 2024-11-14T14:41:09,858 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e2ab5762f8d8f31683b4f4994048f3e, ASSIGN 2024-11-14T14:41:09,858 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=177291d3bf2c8a0b1e9a3689929e9e50, ASSIGN 2024-11-14T14:41:09,858 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=550a3973a29751310a867312ba0b03d1, ASSIGN 2024-11-14T14:41:09,858 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7030563781ca9d5e4d044b06d29712c, server=da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:09,859 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,859 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,859 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,859 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,859 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,40103,1731595261206 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,859 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,859 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,859 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,859 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN 2024-11-14T14:41:09,859 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,859 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN 2024-11-14T14:41:09,859 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,859 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN 2024-11-14T14:41:09,859 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76be481b709a73fdf12e38c88e45d041, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,859 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76be481b709a73fdf12e38c88e45d041, ASSIGN 2024-11-14T14:41:09,859 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=135a286d1aafce40676bb4d4370c9daf, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,859 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=135a286d1aafce40676bb4d4370c9daf, ASSIGN 2024-11-14T14:41:09,859 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e2ab5762f8d8f31683b4f4994048f3e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,859 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e2ab5762f8d8f31683b4f4994048f3e, ASSIGN 2024-11-14T14:41:09,859 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=177291d3bf2c8a0b1e9a3689929e9e50, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,859 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=177291d3bf2c8a0b1e9a3689929e9e50, ASSIGN 2024-11-14T14:41:09,859 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=550a3973a29751310a867312ba0b03d1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,859 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=550a3973a29751310a867312ba0b03d1, ASSIGN 2024-11-14T14:41:09,859 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7030563781ca9d5e4d044b06d29712c, server=da1c6afcd1f9,40103,1731595261206 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,859 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-11-14T14:41:09,867 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=61 2024-11-14T14:41:09,867 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=da1c6afcd1f9,40103,1731595261206, table=hbase:meta, region=1588230740 2024-11-14T14:41:09,868 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as da1c6afcd1f9,40103,1731595261206, state=OPEN 2024-11-14T14:41:09,873 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40103-0x101a73a44750001, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:09,873 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46229-0x101a73a44750003, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:09,873 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34057-0x101a73a44750000, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:09,874 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:09,874 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:09,874 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:09,874 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN to state=OFFLINE, location=null, table=test, region=d35b9113ecfdcde0da6d3c51d6891527 to restore RIT 2024-11-14T14:41:09,874 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN to state=OFFLINE, location=null, table=test, region=b7030563781ca9d5e4d044b06d29712c to restore RIT 2024-11-14T14:41:09,875 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN to state=OFFLINE, location=null, table=test, region=77f677c4c4cfcadb020d2436881cdeaa to restore RIT 2024-11-14T14:41:09,875 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76be481b709a73fdf12e38c88e45d041, ASSIGN to state=OFFLINE, location=null, table=test, region=76be481b709a73fdf12e38c88e45d041 to restore RIT 2024-11-14T14:41:09,875 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=135a286d1aafce40676bb4d4370c9daf, ASSIGN to state=OFFLINE, location=null, table=test, region=135a286d1aafce40676bb4d4370c9daf to restore RIT 2024-11-14T14:41:09,875 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e2ab5762f8d8f31683b4f4994048f3e, ASSIGN to state=OFFLINE, location=null, table=test, region=2e2ab5762f8d8f31683b4f4994048f3e to restore RIT 2024-11-14T14:41:09,875 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=177291d3bf2c8a0b1e9a3689929e9e50, ASSIGN to state=OFFLINE, location=null, table=test, region=177291d3bf2c8a0b1e9a3689929e9e50 to restore RIT 2024-11-14T14:41:09,875 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=550a3973a29751310a867312ba0b03d1, ASSIGN to state=OFFLINE, location=null, table=test, region=550a3973a29751310a867312ba0b03d1 to restore RIT 2024-11-14T14:41:09,885 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731595299885 2024-11-14T14:41:09,886 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-14T14:41:09,886 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T14:41:09,887 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-14T14:41:09,888 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-14T14:41:09,889 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=550a3973a29751310a867312ba0b03d1, ASSIGN 2024-11-14T14:41:09,889 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T14:41:09,889 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=550a3973a29751310a867312ba0b03d1, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T14:41:09,896 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3613f06851ef4465281b91596c2f4aa4, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,897 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7030563781ca9d5e4d044b06d29712c, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,897 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1c1eeeedb7ad89130474f95f0b19f70, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,897 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8dbd58303faefb2f8b5972b0c3fc3aea, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,897 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e71d8bcd86c479813fc8c2e3a01b917, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,898 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d35b9113ecfdcde0da6d3c51d6891527, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,898 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9cb318690a770d1645dfd64bec1a284b, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,898 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad7ad94ee453836a534ae596990b42b8, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,898 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=135a286d1aafce40676bb4d4370c9daf, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,898 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d14b6f516265d1b90f14cf4519cf2b0f, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,899 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d7e464b03c5060f7367637b64f2eae0, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,899 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77f677c4c4cfcadb020d2436881cdeaa, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,899 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9f8e87df66c884919328ba09109d819c, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,899 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f0bff83267f1352a4c903f322542cf14, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,899 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=550a3973a29751310a867312ba0b03d1, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,900 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=624639c373b3612dcb71af916020ab1e, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,900 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e1484a67e6b0eaf08cc5e1d6301f8319, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,900 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b2f6569935d4c7faa1470e22f0b79a20, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,900 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=177291d3bf2c8a0b1e9a3689929e9e50, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,900 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71350ab931ae29ff2210bd9d6c3eeac5, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,901 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d725ecb42ad6f19c89d6ddaa229edc8, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,901 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e2ab5762f8d8f31683b4f4994048f3e, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,901 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbc3f9fabaa1bd9fca0801fe6494782e, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,901 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=954bd925c25bfdb6e075a7471bbb1c4d, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,901 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76be481b709a73fdf12e38c88e45d041, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,901 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1316438481dbf11d64fb665f6cdd4eb6, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,901 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-14T14:41:09,902 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731595329901 2024-11-14T14:41:09,902 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731595389902 2024-11-14T14:41:09,902 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 16 msec 2024-11-14T14:41:09,902 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-14T14:41:09,902 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T14:41:09,902 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T14:41:09,902 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T14:41:09,904 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-14T14:41:09,904 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-14T14:41:09,904 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-14T14:41:09,904 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T14:41:09,905 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-14T14:41:09,909 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-14T14:41:09,909 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,909 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,909 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,909 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,909 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,909 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,909 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,909 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,909 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T14:41:09,909 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T14:41:09,909 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,909 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T14:41:09,909 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T14:41:09,909 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T14:41:09,909 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T14:41:09,914 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-14T14:41:09,917 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true 2024-11-14T14:41:09,918 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:09,918 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:09,918 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T14:41:09,919 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:09,919 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:09,919 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:09,919 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN 2024-11-14T14:41:09,919 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN 2024-11-14T14:41:09,920 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN 2024-11-14T14:41:09,920 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76be481b709a73fdf12e38c88e45d041, ASSIGN 2024-11-14T14:41:09,920 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=135a286d1aafce40676bb4d4370c9daf, ASSIGN 2024-11-14T14:41:09,920 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e2ab5762f8d8f31683b4f4994048f3e, ASSIGN 2024-11-14T14:41:09,920 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=177291d3bf2c8a0b1e9a3689929e9e50, ASSIGN 2024-11-14T14:41:09,920 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=550a3973a29751310a867312ba0b03d1, ASSIGN 2024-11-14T14:41:09,921 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,921 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,922 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,922 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,922 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,40103,1731595261206 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,922 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,922 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,922 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,922 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN 2024-11-14T14:41:09,922 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,922 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN 2024-11-14T14:41:09,923 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,923 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN 2024-11-14T14:41:09,923 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76be481b709a73fdf12e38c88e45d041, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,923 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76be481b709a73fdf12e38c88e45d041, ASSIGN 2024-11-14T14:41:09,923 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=135a286d1aafce40676bb4d4370c9daf, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,923 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=135a286d1aafce40676bb4d4370c9daf, ASSIGN 2024-11-14T14:41:09,923 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e2ab5762f8d8f31683b4f4994048f3e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,923 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e2ab5762f8d8f31683b4f4994048f3e, ASSIGN 2024-11-14T14:41:09,923 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=177291d3bf2c8a0b1e9a3689929e9e50, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,923 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=177291d3bf2c8a0b1e9a3689929e9e50, ASSIGN 2024-11-14T14:41:09,923 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=550a3973a29751310a867312ba0b03d1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,923 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=550a3973a29751310a867312ba0b03d1, ASSIGN 2024-11-14T14:41:09,923 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 17 msec 2024-11-14T14:41:09,947 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=61 2024-11-14T14:41:09,947 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34057 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T14:41:09,947 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=da1c6afcd1f9,40103,1731595261206, table=hbase:meta, region=1588230740 2024-11-14T14:41:09,948 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as da1c6afcd1f9,40103,1731595261206, state=OPEN 2024-11-14T14:41:09,950 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34057-0x101a73a44750000, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:09,950 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40103-0x101a73a44750001, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:09,950 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:09,950 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46229-0x101a73a44750003, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:09,950 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:09,951 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:09,951 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN to state=OFFLINE, location=null, table=test, region=d35b9113ecfdcde0da6d3c51d6891527 to restore RIT 2024-11-14T14:41:09,951 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN to state=OFFLINE, location=null, table=test, region=b7030563781ca9d5e4d044b06d29712c to restore RIT 2024-11-14T14:41:09,952 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN to state=OFFLINE, location=null, table=test, region=77f677c4c4cfcadb020d2436881cdeaa to restore RIT 2024-11-14T14:41:09,952 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76be481b709a73fdf12e38c88e45d041, ASSIGN to state=OFFLINE, location=null, table=test, region=76be481b709a73fdf12e38c88e45d041 to restore RIT 2024-11-14T14:41:09,952 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=135a286d1aafce40676bb4d4370c9daf, ASSIGN to state=OFFLINE, location=null, table=test, region=135a286d1aafce40676bb4d4370c9daf to restore RIT 2024-11-14T14:41:09,952 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e2ab5762f8d8f31683b4f4994048f3e, ASSIGN to state=OFFLINE, location=null, table=test, region=2e2ab5762f8d8f31683b4f4994048f3e to restore RIT 2024-11-14T14:41:09,952 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=177291d3bf2c8a0b1e9a3689929e9e50, ASSIGN to state=OFFLINE, location=null, table=test, region=177291d3bf2c8a0b1e9a3689929e9e50 to restore RIT 2024-11-14T14:41:09,952 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=550a3973a29751310a867312ba0b03d1, ASSIGN to state=OFFLINE, location=null, table=test, region=550a3973a29751310a867312ba0b03d1 to restore RIT 2024-11-14T14:41:09,968 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731595299968 2024-11-14T14:41:09,968 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-14T14:41:09,968 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T14:41:09,970 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-14T14:41:09,977 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-14T14:41:09,977 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=177291d3bf2c8a0b1e9a3689929e9e50, ASSIGN 2024-11-14T14:41:09,977 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T14:41:09,978 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=177291d3bf2c8a0b1e9a3689929e9e50, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T14:41:09,979 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3613f06851ef4465281b91596c2f4aa4, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,980 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7030563781ca9d5e4d044b06d29712c, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,980 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1c1eeeedb7ad89130474f95f0b19f70, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,980 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8dbd58303faefb2f8b5972b0c3fc3aea, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,980 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e71d8bcd86c479813fc8c2e3a01b917, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,980 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d35b9113ecfdcde0da6d3c51d6891527, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,981 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9cb318690a770d1645dfd64bec1a284b, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,981 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad7ad94ee453836a534ae596990b42b8, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,981 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=135a286d1aafce40676bb4d4370c9daf, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,981 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d14b6f516265d1b90f14cf4519cf2b0f, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,981 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d7e464b03c5060f7367637b64f2eae0, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,981 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77f677c4c4cfcadb020d2436881cdeaa, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,982 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9f8e87df66c884919328ba09109d819c, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,982 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f0bff83267f1352a4c903f322542cf14, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,982 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=550a3973a29751310a867312ba0b03d1, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,982 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=624639c373b3612dcb71af916020ab1e, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,982 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e1484a67e6b0eaf08cc5e1d6301f8319, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,982 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b2f6569935d4c7faa1470e22f0b79a20, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,982 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=177291d3bf2c8a0b1e9a3689929e9e50, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,983 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71350ab931ae29ff2210bd9d6c3eeac5, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,983 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d725ecb42ad6f19c89d6ddaa229edc8, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,983 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e2ab5762f8d8f31683b4f4994048f3e, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,983 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbc3f9fabaa1bd9fca0801fe6494782e, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,983 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=954bd925c25bfdb6e075a7471bbb1c4d, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:09,983 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76be481b709a73fdf12e38c88e45d041, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,983 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1316438481dbf11d64fb665f6cdd4eb6, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:09,983 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-14T14:41:09,984 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731595329983 2024-11-14T14:41:09,984 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731595389984 2024-11-14T14:41:09,984 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 15 msec 2024-11-14T14:41:09,984 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-14T14:41:09,984 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T14:41:09,984 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T14:41:09,984 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T14:41:09,985 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-14T14:41:09,985 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-14T14:41:09,985 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-14T14:41:09,986 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T14:41:09,986 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-14T14:41:09,989 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-14T14:41:09,989 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,989 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,989 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,989 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,989 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,989 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,989 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,989 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T14:41:09,989 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T14:41:09,991 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:09,991 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T14:41:09,991 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T14:41:09,991 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T14:41:09,991 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T14:41:09,993 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-14T14:41:09,998 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true 2024-11-14T14:41:09,998 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:09,998 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:09,998 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T14:41:09,998 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:09,998 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:09,999 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:09,999 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN 2024-11-14T14:41:09,999 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN 2024-11-14T14:41:09,999 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN 2024-11-14T14:41:09,999 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76be481b709a73fdf12e38c88e45d041, ASSIGN 2024-11-14T14:41:09,999 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=135a286d1aafce40676bb4d4370c9daf, ASSIGN 2024-11-14T14:41:09,999 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e2ab5762f8d8f31683b4f4994048f3e, ASSIGN 2024-11-14T14:41:09,999 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=177291d3bf2c8a0b1e9a3689929e9e50, ASSIGN 2024-11-14T14:41:09,999 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,999 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,999 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,999 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,999 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,40103,1731595261206 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,999 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,999 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:09,999 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,999 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN 2024-11-14T14:41:09,999 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,999 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN 2024-11-14T14:41:09,999 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,999 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN 2024-11-14T14:41:09,999 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76be481b709a73fdf12e38c88e45d041, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:09,999 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76be481b709a73fdf12e38c88e45d041, ASSIGN 2024-11-14T14:41:10,000 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=135a286d1aafce40676bb4d4370c9daf, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:10,000 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=135a286d1aafce40676bb4d4370c9daf, ASSIGN 2024-11-14T14:41:10,000 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e2ab5762f8d8f31683b4f4994048f3e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:10,000 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e2ab5762f8d8f31683b4f4994048f3e, ASSIGN 2024-11-14T14:41:10,000 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=177291d3bf2c8a0b1e9a3689929e9e50, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:10,000 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=177291d3bf2c8a0b1e9a3689929e9e50, ASSIGN 2024-11-14T14:41:10,000 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 13 msec 2024-11-14T14:41:10,014 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=61 2024-11-14T14:41:10,014 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=da1c6afcd1f9,40103,1731595261206, table=hbase:meta, region=1588230740 2024-11-14T14:41:10,014 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as da1c6afcd1f9,40103,1731595261206, state=OPEN 2024-11-14T14:41:10,017 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46229-0x101a73a44750003, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:10,017 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34057-0x101a73a44750000, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:10,017 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40103-0x101a73a44750001, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:10,017 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:10,017 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:10,017 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:10,019 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN to state=OFFLINE, location=null, table=test, region=d35b9113ecfdcde0da6d3c51d6891527 to restore RIT 2024-11-14T14:41:10,019 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN to state=OFFLINE, location=null, table=test, region=b7030563781ca9d5e4d044b06d29712c to restore RIT 2024-11-14T14:41:10,019 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN to state=OFFLINE, location=null, table=test, region=77f677c4c4cfcadb020d2436881cdeaa to restore RIT 2024-11-14T14:41:10,019 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76be481b709a73fdf12e38c88e45d041, ASSIGN to state=OFFLINE, location=null, table=test, region=76be481b709a73fdf12e38c88e45d041 to restore RIT 2024-11-14T14:41:10,019 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=135a286d1aafce40676bb4d4370c9daf, ASSIGN to state=OFFLINE, location=null, table=test, region=135a286d1aafce40676bb4d4370c9daf to restore RIT 2024-11-14T14:41:10,019 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e2ab5762f8d8f31683b4f4994048f3e, ASSIGN to state=OFFLINE, location=null, table=test, region=2e2ab5762f8d8f31683b4f4994048f3e to restore RIT 2024-11-14T14:41:10,019 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=177291d3bf2c8a0b1e9a3689929e9e50, ASSIGN to state=OFFLINE, location=null, table=test, region=177291d3bf2c8a0b1e9a3689929e9e50 to restore RIT 2024-11-14T14:41:10,037 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731595300037 2024-11-14T14:41:10,037 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-14T14:41:10,039 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T14:41:10,045 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-14T14:41:10,045 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3613f06851ef4465281b91596c2f4aa4, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,046 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7030563781ca9d5e4d044b06d29712c, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,046 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1c1eeeedb7ad89130474f95f0b19f70, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,046 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8dbd58303faefb2f8b5972b0c3fc3aea, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,046 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e71d8bcd86c479813fc8c2e3a01b917, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,046 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d35b9113ecfdcde0da6d3c51d6891527, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,047 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9cb318690a770d1645dfd64bec1a284b, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,047 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad7ad94ee453836a534ae596990b42b8, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,047 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=135a286d1aafce40676bb4d4370c9daf, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,047 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d14b6f516265d1b90f14cf4519cf2b0f, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,047 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d7e464b03c5060f7367637b64f2eae0, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,048 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77f677c4c4cfcadb020d2436881cdeaa, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,048 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9f8e87df66c884919328ba09109d819c, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,048 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f0bff83267f1352a4c903f322542cf14, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,048 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=550a3973a29751310a867312ba0b03d1, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,048 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-14T14:41:10,048 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e2ab5762f8d8f31683b4f4994048f3e, ASSIGN 2024-11-14T14:41:10,048 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T14:41:10,048 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=624639c373b3612dcb71af916020ab1e, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,048 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e2ab5762f8d8f31683b4f4994048f3e, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T14:41:10,049 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e1484a67e6b0eaf08cc5e1d6301f8319, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,049 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b2f6569935d4c7faa1470e22f0b79a20, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,049 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=177291d3bf2c8a0b1e9a3689929e9e50, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,049 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71350ab931ae29ff2210bd9d6c3eeac5, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,049 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d725ecb42ad6f19c89d6ddaa229edc8, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,049 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e2ab5762f8d8f31683b4f4994048f3e, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,050 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbc3f9fabaa1bd9fca0801fe6494782e, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,050 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=954bd925c25bfdb6e075a7471bbb1c4d, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,050 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76be481b709a73fdf12e38c88e45d041, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,050 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1316438481dbf11d64fb665f6cdd4eb6, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,050 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-14T14:41:10,050 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731595330050 2024-11-14T14:41:10,050 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731595390050 2024-11-14T14:41:10,050 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 13 msec 2024-11-14T14:41:10,050 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-14T14:41:10,050 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T14:41:10,051 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T14:41:10,051 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T14:41:10,051 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-14T14:41:10,051 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-14T14:41:10,051 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-14T14:41:10,051 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T14:41:10,052 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-14T14:41:10,053 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34057 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T14:41:10,054 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-14T14:41:10,054 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:10,054 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:10,054 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:10,054 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:10,054 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:10,055 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:10,055 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T14:41:10,055 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T14:41:10,055 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:10,055 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T14:41:10,055 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T14:41:10,055 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T14:41:10,055 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T14:41:10,056 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-14T14:41:10,056 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34057 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T14:41:10,057 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true 2024-11-14T14:41:10,057 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:10,057 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:10,057 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T14:41:10,057 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:10,057 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:10,057 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:10,057 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN 2024-11-14T14:41:10,057 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN 2024-11-14T14:41:10,057 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN 2024-11-14T14:41:10,057 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76be481b709a73fdf12e38c88e45d041, ASSIGN 2024-11-14T14:41:10,058 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=135a286d1aafce40676bb4d4370c9daf, ASSIGN 2024-11-14T14:41:10,058 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e2ab5762f8d8f31683b4f4994048f3e, ASSIGN 2024-11-14T14:41:10,058 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:10,058 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:10,058 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:10,058 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:10,058 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,40103,1731595261206 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:10,058 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:10,058 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:10,058 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:10,058 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN 2024-11-14T14:41:10,058 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:10,058 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN 2024-11-14T14:41:10,058 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:10,058 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN 2024-11-14T14:41:10,058 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76be481b709a73fdf12e38c88e45d041, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:10,058 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76be481b709a73fdf12e38c88e45d041, ASSIGN 2024-11-14T14:41:10,058 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=135a286d1aafce40676bb4d4370c9daf, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:10,058 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=135a286d1aafce40676bb4d4370c9daf, ASSIGN 2024-11-14T14:41:10,058 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e2ab5762f8d8f31683b4f4994048f3e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:10,058 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e2ab5762f8d8f31683b4f4994048f3e, ASSIGN 2024-11-14T14:41:10,058 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-14T14:41:10,064 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=61 2024-11-14T14:41:10,064 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=da1c6afcd1f9,40103,1731595261206, table=hbase:meta, region=1588230740 2024-11-14T14:41:10,065 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as da1c6afcd1f9,40103,1731595261206, state=OPEN 2024-11-14T14:41:10,067 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40103-0x101a73a44750001, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:10,067 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34057-0x101a73a44750000, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:10,067 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46229-0x101a73a44750003, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:10,067 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:10,067 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:10,068 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:10,070 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN to state=OFFLINE, location=null, table=test, region=d35b9113ecfdcde0da6d3c51d6891527 to restore RIT 2024-11-14T14:41:10,070 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN to state=OFFLINE, location=null, table=test, region=b7030563781ca9d5e4d044b06d29712c to restore RIT 2024-11-14T14:41:10,070 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN to state=OFFLINE, location=null, table=test, region=77f677c4c4cfcadb020d2436881cdeaa to restore RIT 2024-11-14T14:41:10,070 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76be481b709a73fdf12e38c88e45d041, ASSIGN to state=OFFLINE, location=null, table=test, region=76be481b709a73fdf12e38c88e45d041 to restore RIT 2024-11-14T14:41:10,070 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=135a286d1aafce40676bb4d4370c9daf, ASSIGN to state=OFFLINE, location=null, table=test, region=135a286d1aafce40676bb4d4370c9daf to restore RIT 2024-11-14T14:41:10,070 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e2ab5762f8d8f31683b4f4994048f3e, ASSIGN to state=OFFLINE, location=null, table=test, region=2e2ab5762f8d8f31683b4f4994048f3e to restore RIT 2024-11-14T14:41:10,085 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731595300085 2024-11-14T14:41:10,085 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-14T14:41:10,086 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T14:41:10,088 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-14T14:41:10,093 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3613f06851ef4465281b91596c2f4aa4, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,093 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7030563781ca9d5e4d044b06d29712c, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,093 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1c1eeeedb7ad89130474f95f0b19f70, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,093 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8dbd58303faefb2f8b5972b0c3fc3aea, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,093 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e71d8bcd86c479813fc8c2e3a01b917, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,094 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d35b9113ecfdcde0da6d3c51d6891527, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,094 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9cb318690a770d1645dfd64bec1a284b, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,094 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad7ad94ee453836a534ae596990b42b8, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,094 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=135a286d1aafce40676bb4d4370c9daf, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,094 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d14b6f516265d1b90f14cf4519cf2b0f, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,094 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d7e464b03c5060f7367637b64f2eae0, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,095 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77f677c4c4cfcadb020d2436881cdeaa, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,095 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9f8e87df66c884919328ba09109d819c, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,095 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f0bff83267f1352a4c903f322542cf14, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,095 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=550a3973a29751310a867312ba0b03d1, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,095 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=624639c373b3612dcb71af916020ab1e, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,095 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e1484a67e6b0eaf08cc5e1d6301f8319, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,095 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b2f6569935d4c7faa1470e22f0b79a20, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,095 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=177291d3bf2c8a0b1e9a3689929e9e50, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,096 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71350ab931ae29ff2210bd9d6c3eeac5, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,096 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d725ecb42ad6f19c89d6ddaa229edc8, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,096 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e2ab5762f8d8f31683b4f4994048f3e, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,096 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbc3f9fabaa1bd9fca0801fe6494782e, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,096 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=954bd925c25bfdb6e075a7471bbb1c4d, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,096 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76be481b709a73fdf12e38c88e45d041, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,097 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1316438481dbf11d64fb665f6cdd4eb6, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,097 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-14T14:41:10,097 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=135a286d1aafce40676bb4d4370c9daf, ASSIGN 2024-11-14T14:41:10,097 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T14:41:10,097 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=135a286d1aafce40676bb4d4370c9daf, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T14:41:10,100 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-14T14:41:10,100 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731595330100 2024-11-14T14:41:10,100 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731595390100 2024-11-14T14:41:10,100 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 15 msec 2024-11-14T14:41:10,100 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-14T14:41:10,100 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T14:41:10,100 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T14:41:10,100 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T14:41:10,101 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-14T14:41:10,101 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-14T14:41:10,101 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-14T14:41:10,101 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T14:41:10,102 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-14T14:41:10,104 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-14T14:41:10,104 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:10,104 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:10,104 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:10,104 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:10,104 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:10,104 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T14:41:10,104 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T14:41:10,104 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:10,104 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T14:41:10,104 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T14:41:10,104 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T14:41:10,104 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T14:41:10,105 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-14T14:41:10,106 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true 2024-11-14T14:41:10,106 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:10,106 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:10,106 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T14:41:10,106 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:10,106 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:10,106 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:10,107 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN 2024-11-14T14:41:10,107 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN 2024-11-14T14:41:10,107 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN 2024-11-14T14:41:10,107 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76be481b709a73fdf12e38c88e45d041, ASSIGN 2024-11-14T14:41:10,107 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=135a286d1aafce40676bb4d4370c9daf, ASSIGN 2024-11-14T14:41:10,107 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:10,107 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:10,107 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:10,107 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:10,107 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,40103,1731595261206 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:10,107 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:10,107 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:10,107 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:10,107 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN 2024-11-14T14:41:10,107 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:10,107 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN 2024-11-14T14:41:10,107 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:10,108 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN 2024-11-14T14:41:10,108 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76be481b709a73fdf12e38c88e45d041, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:10,108 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76be481b709a73fdf12e38c88e45d041, ASSIGN 2024-11-14T14:41:10,108 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=135a286d1aafce40676bb4d4370c9daf, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:10,108 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=135a286d1aafce40676bb4d4370c9daf, ASSIGN 2024-11-14T14:41:10,108 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-14T14:41:10,110 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=61 2024-11-14T14:41:10,110 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=da1c6afcd1f9,40103,1731595261206, table=hbase:meta, region=1588230740 2024-11-14T14:41:10,110 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as da1c6afcd1f9,40103,1731595261206, state=OPEN 2024-11-14T14:41:10,112 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34057-0x101a73a44750000, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:10,112 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40103-0x101a73a44750001, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:10,112 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46229-0x101a73a44750003, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:10,112 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:10,112 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:10,112 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:10,113 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN to state=OFFLINE, location=null, table=test, region=d35b9113ecfdcde0da6d3c51d6891527 to restore RIT 2024-11-14T14:41:10,113 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN to state=OFFLINE, location=null, table=test, region=b7030563781ca9d5e4d044b06d29712c to restore RIT 2024-11-14T14:41:10,114 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN to state=OFFLINE, location=null, table=test, region=77f677c4c4cfcadb020d2436881cdeaa to restore RIT 2024-11-14T14:41:10,114 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76be481b709a73fdf12e38c88e45d041, ASSIGN to state=OFFLINE, location=null, table=test, region=76be481b709a73fdf12e38c88e45d041 to restore RIT 2024-11-14T14:41:10,114 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=135a286d1aafce40676bb4d4370c9daf, ASSIGN to state=OFFLINE, location=null, table=test, region=135a286d1aafce40676bb4d4370c9daf to restore RIT 2024-11-14T14:41:10,141 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731595300141 2024-11-14T14:41:10,141 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-14T14:41:10,142 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T14:41:10,145 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-14T14:41:10,147 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3613f06851ef4465281b91596c2f4aa4, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,147 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-14T14:41:10,147 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76be481b709a73fdf12e38c88e45d041, ASSIGN 2024-11-14T14:41:10,147 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T14:41:10,147 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7030563781ca9d5e4d044b06d29712c, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,147 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76be481b709a73fdf12e38c88e45d041, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T14:41:10,148 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1c1eeeedb7ad89130474f95f0b19f70, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,148 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8dbd58303faefb2f8b5972b0c3fc3aea, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,148 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e71d8bcd86c479813fc8c2e3a01b917, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,148 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d35b9113ecfdcde0da6d3c51d6891527, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,148 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9cb318690a770d1645dfd64bec1a284b, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,148 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad7ad94ee453836a534ae596990b42b8, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,149 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=135a286d1aafce40676bb4d4370c9daf, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,149 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d14b6f516265d1b90f14cf4519cf2b0f, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,149 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d7e464b03c5060f7367637b64f2eae0, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,149 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77f677c4c4cfcadb020d2436881cdeaa, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,149 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9f8e87df66c884919328ba09109d819c, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,149 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f0bff83267f1352a4c903f322542cf14, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=550a3973a29751310a867312ba0b03d1, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=624639c373b3612dcb71af916020ab1e, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e1484a67e6b0eaf08cc5e1d6301f8319, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b2f6569935d4c7faa1470e22f0b79a20, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=177291d3bf2c8a0b1e9a3689929e9e50, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71350ab931ae29ff2210bd9d6c3eeac5, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,151 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d725ecb42ad6f19c89d6ddaa229edc8, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,151 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e2ab5762f8d8f31683b4f4994048f3e, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,151 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbc3f9fabaa1bd9fca0801fe6494782e, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,151 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=954bd925c25bfdb6e075a7471bbb1c4d, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,151 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76be481b709a73fdf12e38c88e45d041, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,151 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1316438481dbf11d64fb665f6cdd4eb6, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,151 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-14T14:41:10,152 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731595330151 2024-11-14T14:41:10,152 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731595390152 2024-11-14T14:41:10,152 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-11-14T14:41:10,152 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-14T14:41:10,152 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T14:41:10,152 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T14:41:10,152 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T14:41:10,152 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-14T14:41:10,152 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-14T14:41:10,152 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-14T14:41:10,153 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T14:41:10,154 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-14T14:41:10,156 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-14T14:41:10,156 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:10,156 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:10,156 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:10,156 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:10,156 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T14:41:10,156 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T14:41:10,156 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:10,156 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T14:41:10,156 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T14:41:10,156 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T14:41:10,156 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T14:41:10,157 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-14T14:41:10,159 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true 2024-11-14T14:41:10,159 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:10,159 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:10,159 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T14:41:10,159 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:10,159 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:10,159 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:10,159 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN 2024-11-14T14:41:10,159 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN 2024-11-14T14:41:10,159 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN 2024-11-14T14:41:10,159 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76be481b709a73fdf12e38c88e45d041, ASSIGN 2024-11-14T14:41:10,159 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:10,159 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:10,159 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:10,159 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:10,159 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,40103,1731595261206 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:10,159 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:10,159 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34057 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T14:41:10,159 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:10,159 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:10,159 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN 2024-11-14T14:41:10,159 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:10,159 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN 2024-11-14T14:41:10,159 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:10,159 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN 2024-11-14T14:41:10,159 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76be481b709a73fdf12e38c88e45d041, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:10,159 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76be481b709a73fdf12e38c88e45d041, ASSIGN 2024-11-14T14:41:10,160 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-14T14:41:10,160 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34057 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T14:41:10,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=61 2024-11-14T14:41:10,160 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=da1c6afcd1f9,40103,1731595261206, table=hbase:meta, region=1588230740 2024-11-14T14:41:10,161 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as da1c6afcd1f9,40103,1731595261206, state=OPEN 2024-11-14T14:41:10,163 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40103-0x101a73a44750001, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:10,163 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46229-0x101a73a44750003, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:10,163 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34057-0x101a73a44750000, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:10,163 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:10,163 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:10,163 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:10,164 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN to state=OFFLINE, location=null, table=test, region=d35b9113ecfdcde0da6d3c51d6891527 to restore RIT 2024-11-14T14:41:10,164 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN to state=OFFLINE, location=null, table=test, region=b7030563781ca9d5e4d044b06d29712c to restore RIT 2024-11-14T14:41:10,164 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN to state=OFFLINE, location=null, table=test, region=77f677c4c4cfcadb020d2436881cdeaa to restore RIT 2024-11-14T14:41:10,164 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=76be481b709a73fdf12e38c88e45d041, ASSIGN to state=OFFLINE, location=null, table=test, region=76be481b709a73fdf12e38c88e45d041 to restore RIT 2024-11-14T14:41:10,196 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731595300196 2024-11-14T14:41:10,196 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-14T14:41:10,202 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T14:41:10,204 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-14T14:41:10,205 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-14T14:41:10,205 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN 2024-11-14T14:41:10,205 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T14:41:10,206 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T14:41:10,207 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3613f06851ef4465281b91596c2f4aa4, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,207 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7030563781ca9d5e4d044b06d29712c, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,208 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1c1eeeedb7ad89130474f95f0b19f70, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,208 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8dbd58303faefb2f8b5972b0c3fc3aea, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,208 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e71d8bcd86c479813fc8c2e3a01b917, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,208 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d35b9113ecfdcde0da6d3c51d6891527, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,208 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9cb318690a770d1645dfd64bec1a284b, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,208 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad7ad94ee453836a534ae596990b42b8, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,208 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=135a286d1aafce40676bb4d4370c9daf, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,209 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d14b6f516265d1b90f14cf4519cf2b0f, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,209 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d7e464b03c5060f7367637b64f2eae0, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,209 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77f677c4c4cfcadb020d2436881cdeaa, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,209 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9f8e87df66c884919328ba09109d819c, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,209 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f0bff83267f1352a4c903f322542cf14, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,209 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=550a3973a29751310a867312ba0b03d1, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,209 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=624639c373b3612dcb71af916020ab1e, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,209 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e1484a67e6b0eaf08cc5e1d6301f8319, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,210 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b2f6569935d4c7faa1470e22f0b79a20, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,210 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=177291d3bf2c8a0b1e9a3689929e9e50, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,210 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71350ab931ae29ff2210bd9d6c3eeac5, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,210 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d725ecb42ad6f19c89d6ddaa229edc8, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,210 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e2ab5762f8d8f31683b4f4994048f3e, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,210 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbc3f9fabaa1bd9fca0801fe6494782e, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,211 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=954bd925c25bfdb6e075a7471bbb1c4d, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,211 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76be481b709a73fdf12e38c88e45d041, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,211 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1316438481dbf11d64fb665f6cdd4eb6, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,211 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-14T14:41:10,211 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731595330211 2024-11-14T14:41:10,211 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731595390211 2024-11-14T14:41:10,211 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 15 msec 2024-11-14T14:41:10,211 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-14T14:41:10,211 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T14:41:10,211 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T14:41:10,211 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T14:41:10,212 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-14T14:41:10,212 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-14T14:41:10,212 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-14T14:41:10,212 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T14:41:10,213 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-14T14:41:10,215 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-14T14:41:10,215 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:10,215 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:10,215 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:10,215 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T14:41:10,215 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T14:41:10,215 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:10,215 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T14:41:10,215 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T14:41:10,215 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T14:41:10,216 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T14:41:10,216 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-14T14:41:10,218 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true 2024-11-14T14:41:10,218 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:10,218 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:10,218 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T14:41:10,218 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:10,218 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:10,218 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:10,218 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN 2024-11-14T14:41:10,218 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN 2024-11-14T14:41:10,218 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN 2024-11-14T14:41:10,218 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:10,218 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:10,218 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:10,218 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:10,218 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,40103,1731595261206 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:10,218 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:10,218 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:10,218 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:10,218 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN 2024-11-14T14:41:10,218 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:10,218 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN 2024-11-14T14:41:10,218 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:10,218 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN 2024-11-14T14:41:10,219 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-14T14:41:10,219 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=61 2024-11-14T14:41:10,220 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=da1c6afcd1f9,40103,1731595261206, table=hbase:meta, region=1588230740 2024-11-14T14:41:10,220 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as da1c6afcd1f9,40103,1731595261206, state=OPEN 2024-11-14T14:41:10,222 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40103-0x101a73a44750001, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:10,222 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46229-0x101a73a44750003, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:10,222 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:10,222 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:10,223 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34057-0x101a73a44750000, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:10,223 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:10,224 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN to state=OFFLINE, location=null, table=test, region=d35b9113ecfdcde0da6d3c51d6891527 to restore RIT 2024-11-14T14:41:10,224 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN to state=OFFLINE, location=null, table=test, region=b7030563781ca9d5e4d044b06d29712c to restore RIT 2024-11-14T14:41:10,224 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=77f677c4c4cfcadb020d2436881cdeaa, ASSIGN to state=OFFLINE, location=null, table=test, region=77f677c4c4cfcadb020d2436881cdeaa to restore RIT 2024-11-14T14:41:10,227 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731595300227 2024-11-14T14:41:10,227 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-14T14:41:10,229 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T14:41:10,230 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-14T14:41:10,233 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3613f06851ef4465281b91596c2f4aa4, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,233 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7030563781ca9d5e4d044b06d29712c, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,233 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1c1eeeedb7ad89130474f95f0b19f70, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,233 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8dbd58303faefb2f8b5972b0c3fc3aea, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,234 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e71d8bcd86c479813fc8c2e3a01b917, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,234 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d35b9113ecfdcde0da6d3c51d6891527, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,234 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9cb318690a770d1645dfd64bec1a284b, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,234 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad7ad94ee453836a534ae596990b42b8, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,234 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=135a286d1aafce40676bb4d4370c9daf, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,234 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d14b6f516265d1b90f14cf4519cf2b0f, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,234 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d7e464b03c5060f7367637b64f2eae0, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,235 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77f677c4c4cfcadb020d2436881cdeaa, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,235 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9f8e87df66c884919328ba09109d819c, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,235 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f0bff83267f1352a4c903f322542cf14, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,235 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=550a3973a29751310a867312ba0b03d1, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,235 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=624639c373b3612dcb71af916020ab1e, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,235 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e1484a67e6b0eaf08cc5e1d6301f8319, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,235 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b2f6569935d4c7faa1470e22f0b79a20, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,235 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=177291d3bf2c8a0b1e9a3689929e9e50, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,236 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71350ab931ae29ff2210bd9d6c3eeac5, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,236 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d725ecb42ad6f19c89d6ddaa229edc8, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,236 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e2ab5762f8d8f31683b4f4994048f3e, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,236 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbc3f9fabaa1bd9fca0801fe6494782e, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,236 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=954bd925c25bfdb6e075a7471bbb1c4d, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,236 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76be481b709a73fdf12e38c88e45d041, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,236 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1316438481dbf11d64fb665f6cdd4eb6, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,236 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-14T14:41:10,236 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731595330236 2024-11-14T14:41:10,236 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-14T14:41:10,237 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731595390236 2024-11-14T14:41:10,237 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN 2024-11-14T14:41:10,237 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T14:41:10,238 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T14:41:10,237 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-14T14:41:10,239 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-14T14:41:10,239 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T14:41:10,239 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T14:41:10,239 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T14:41:10,245 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-14T14:41:10,245 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-14T14:41:10,245 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-14T14:41:10,246 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T14:41:10,247 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-14T14:41:10,251 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-14T14:41:10,251 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:10,251 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:10,251 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T14:41:10,251 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T14:41:10,251 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:10,252 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T14:41:10,252 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T14:41:10,252 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T14:41:10,252 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T14:41:10,253 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-14T14:41:10,254 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true 2024-11-14T14:41:10,254 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:10,254 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:10,254 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T14:41:10,254 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:10,254 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:10,254 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:10,254 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN 2024-11-14T14:41:10,254 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN 2024-11-14T14:41:10,254 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:10,254 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:10,254 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:10,254 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:10,254 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,40103,1731595261206 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:10,254 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:10,254 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:10,255 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:10,255 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN 2024-11-14T14:41:10,255 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:10,255 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN 2024-11-14T14:41:10,255 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-11-14T14:41:10,256 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=61 2024-11-14T14:41:10,256 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=da1c6afcd1f9,40103,1731595261206, table=hbase:meta, region=1588230740 2024-11-14T14:41:10,256 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as da1c6afcd1f9,40103,1731595261206, state=OPEN 2024-11-14T14:41:10,260 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46229-0x101a73a44750003, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:10,260 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40103-0x101a73a44750001, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:10,261 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:10,261 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:10,262 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34057-0x101a73a44750000, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:10,262 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:10,264 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34057 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T14:41:10,265 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34057 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T14:41:10,265 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN to state=OFFLINE, location=null, table=test, region=d35b9113ecfdcde0da6d3c51d6891527 to restore RIT 2024-11-14T14:41:10,265 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b7030563781ca9d5e4d044b06d29712c, ASSIGN to state=OFFLINE, location=null, table=test, region=b7030563781ca9d5e4d044b06d29712c to restore RIT 2024-11-14T14:41:10,276 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731595300276 2024-11-14T14:41:10,276 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-14T14:41:10,276 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T14:41:10,277 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-14T14:41:10,282 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-14T14:41:10,282 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3613f06851ef4465281b91596c2f4aa4, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,282 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN 2024-11-14T14:41:10,282 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T14:41:10,282 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7030563781ca9d5e4d044b06d29712c, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,282 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T14:41:10,282 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1c1eeeedb7ad89130474f95f0b19f70, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,282 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8dbd58303faefb2f8b5972b0c3fc3aea, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,283 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e71d8bcd86c479813fc8c2e3a01b917, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,283 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d35b9113ecfdcde0da6d3c51d6891527, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,283 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9cb318690a770d1645dfd64bec1a284b, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,283 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad7ad94ee453836a534ae596990b42b8, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,283 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=135a286d1aafce40676bb4d4370c9daf, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,283 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d14b6f516265d1b90f14cf4519cf2b0f, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,283 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d7e464b03c5060f7367637b64f2eae0, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,283 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77f677c4c4cfcadb020d2436881cdeaa, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,283 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9f8e87df66c884919328ba09109d819c, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,283 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f0bff83267f1352a4c903f322542cf14, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,283 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=550a3973a29751310a867312ba0b03d1, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,284 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=624639c373b3612dcb71af916020ab1e, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,284 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e1484a67e6b0eaf08cc5e1d6301f8319, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,284 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b2f6569935d4c7faa1470e22f0b79a20, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,284 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=177291d3bf2c8a0b1e9a3689929e9e50, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,284 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71350ab931ae29ff2210bd9d6c3eeac5, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,284 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d725ecb42ad6f19c89d6ddaa229edc8, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,284 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e2ab5762f8d8f31683b4f4994048f3e, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,284 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbc3f9fabaa1bd9fca0801fe6494782e, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,284 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=954bd925c25bfdb6e075a7471bbb1c4d, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,284 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76be481b709a73fdf12e38c88e45d041, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,285 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1316438481dbf11d64fb665f6cdd4eb6, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,285 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-14T14:41:10,285 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731595330285 2024-11-14T14:41:10,285 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731595390285 2024-11-14T14:41:10,285 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-14T14:41:10,285 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-14T14:41:10,285 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T14:41:10,285 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T14:41:10,285 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T14:41:10,285 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-14T14:41:10,285 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-14T14:41:10,285 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-14T14:41:10,286 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T14:41:10,287 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-14T14:41:10,288 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-14T14:41:10,289 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:10,289 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T14:41:10,289 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T14:41:10,289 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:10,289 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T14:41:10,289 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T14:41:10,289 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T14:41:10,289 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T14:41:10,290 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-14T14:41:10,292 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true 2024-11-14T14:41:10,292 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:10,292 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:10,292 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T14:41:10,292 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:10,292 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:10,292 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:10,292 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN 2024-11-14T14:41:10,292 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:10,292 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:10,292 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:10,292 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:10,292 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,40103,1731595261206 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:10,292 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:10,292 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:10,292 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:10,292 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN 2024-11-14T14:41:10,292 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-14T14:41:10,293 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=61 2024-11-14T14:41:10,293 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=da1c6afcd1f9,40103,1731595261206, table=hbase:meta, region=1588230740 2024-11-14T14:41:10,293 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as da1c6afcd1f9,40103,1731595261206, state=OPEN 2024-11-14T14:41:10,295 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34057-0x101a73a44750000, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:10,295 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40103-0x101a73a44750001, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:10,295 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46229-0x101a73a44750003, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:10,296 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:10,296 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:10,296 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:10,297 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d35b9113ecfdcde0da6d3c51d6891527, ASSIGN to state=OFFLINE, location=null, table=test, region=d35b9113ecfdcde0da6d3c51d6891527 to restore RIT 2024-11-14T14:41:10,306 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731595300306 2024-11-14T14:41:10,306 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-14T14:41:10,307 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T14:41:10,309 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-14T14:41:10,311 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-14T14:41:10,311 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:10,311 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T14:41:10,311 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T14:41:10,315 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3613f06851ef4465281b91596c2f4aa4, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,315 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7030563781ca9d5e4d044b06d29712c, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,315 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1c1eeeedb7ad89130474f95f0b19f70, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,315 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8dbd58303faefb2f8b5972b0c3fc3aea, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,315 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e71d8bcd86c479813fc8c2e3a01b917, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,315 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d35b9113ecfdcde0da6d3c51d6891527, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,315 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9cb318690a770d1645dfd64bec1a284b, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,316 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad7ad94ee453836a534ae596990b42b8, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,316 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=135a286d1aafce40676bb4d4370c9daf, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,316 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d14b6f516265d1b90f14cf4519cf2b0f, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,316 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d7e464b03c5060f7367637b64f2eae0, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,316 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77f677c4c4cfcadb020d2436881cdeaa, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,316 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9f8e87df66c884919328ba09109d819c, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,316 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f0bff83267f1352a4c903f322542cf14, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,316 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=550a3973a29751310a867312ba0b03d1, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,317 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=624639c373b3612dcb71af916020ab1e, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,317 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e1484a67e6b0eaf08cc5e1d6301f8319, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,317 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b2f6569935d4c7faa1470e22f0b79a20, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,317 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=177291d3bf2c8a0b1e9a3689929e9e50, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,317 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71350ab931ae29ff2210bd9d6c3eeac5, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,317 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d725ecb42ad6f19c89d6ddaa229edc8, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,317 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e2ab5762f8d8f31683b4f4994048f3e, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,317 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbc3f9fabaa1bd9fca0801fe6494782e, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,317 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=954bd925c25bfdb6e075a7471bbb1c4d, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,317 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76be481b709a73fdf12e38c88e45d041, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,318 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1316438481dbf11d64fb665f6cdd4eb6, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,318 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-14T14:41:10,318 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731595330318 2024-11-14T14:41:10,318 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731595390318 2024-11-14T14:41:10,318 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 12 msec 2024-11-14T14:41:10,318 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-14T14:41:10,318 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T14:41:10,318 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T14:41:10,318 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T14:41:10,318 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-14T14:41:10,318 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-14T14:41:10,318 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-14T14:41:10,319 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T14:41:10,320 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-14T14:41:10,321 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-14T14:41:10,322 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T14:41:10,322 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T14:41:10,322 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:10,322 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T14:41:10,322 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T14:41:10,322 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T14:41:10,322 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T14:41:10,323 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-14T14:41:10,323 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true 2024-11-14T14:41:10,324 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:10,324 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:10,324 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T14:41:10,324 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:10,324 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:10,324 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:10,324 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:10,324 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:10,324 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:10,324 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:10,324 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,40103,1731595261206 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:10,324 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:10,324 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:10,324 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-14T14:41:10,328 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=61 2024-11-14T14:41:10,328 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=da1c6afcd1f9,40103,1731595261206, table=hbase:meta, region=1588230740 2024-11-14T14:41:10,329 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as da1c6afcd1f9,40103,1731595261206, state=OPEN 2024-11-14T14:41:10,331 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34057-0x101a73a44750000, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:10,331 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40103-0x101a73a44750001, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:10,331 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46229-0x101a73a44750003, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:10,331 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:10,331 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:10,331 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:10,337 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731595300337 2024-11-14T14:41:10,337 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-14T14:41:10,339 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T14:41:10,340 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-14T14:41:10,343 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-14T14:41:10,343 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:10,343 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T14:41:10,343 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T14:41:10,344 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3613f06851ef4465281b91596c2f4aa4, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,345 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7030563781ca9d5e4d044b06d29712c, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,345 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1c1eeeedb7ad89130474f95f0b19f70, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,345 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8dbd58303faefb2f8b5972b0c3fc3aea, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,345 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e71d8bcd86c479813fc8c2e3a01b917, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,345 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d35b9113ecfdcde0da6d3c51d6891527, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,345 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9cb318690a770d1645dfd64bec1a284b, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,345 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad7ad94ee453836a534ae596990b42b8, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,345 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=135a286d1aafce40676bb4d4370c9daf, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,345 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d14b6f516265d1b90f14cf4519cf2b0f, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,345 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d7e464b03c5060f7367637b64f2eae0, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,346 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77f677c4c4cfcadb020d2436881cdeaa, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,346 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9f8e87df66c884919328ba09109d819c, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,346 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f0bff83267f1352a4c903f322542cf14, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,346 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=550a3973a29751310a867312ba0b03d1, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,346 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=624639c373b3612dcb71af916020ab1e, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,346 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e1484a67e6b0eaf08cc5e1d6301f8319, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,346 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b2f6569935d4c7faa1470e22f0b79a20, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,346 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=177291d3bf2c8a0b1e9a3689929e9e50, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,346 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71350ab931ae29ff2210bd9d6c3eeac5, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,346 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d725ecb42ad6f19c89d6ddaa229edc8, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,346 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e2ab5762f8d8f31683b4f4994048f3e, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,347 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbc3f9fabaa1bd9fca0801fe6494782e, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,347 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=954bd925c25bfdb6e075a7471bbb1c4d, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,347 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76be481b709a73fdf12e38c88e45d041, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,347 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1316438481dbf11d64fb665f6cdd4eb6, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,347 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-14T14:41:10,347 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731595330347 2024-11-14T14:41:10,347 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731595390347 2024-11-14T14:41:10,347 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-14T14:41:10,347 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-14T14:41:10,347 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T14:41:10,347 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T14:41:10,347 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T14:41:10,348 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-14T14:41:10,348 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-14T14:41:10,348 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-14T14:41:10,348 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T14:41:10,349 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-14T14:41:10,351 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-14T14:41:10,351 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T14:41:10,351 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T14:41:10,352 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:10,352 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T14:41:10,352 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T14:41:10,352 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T14:41:10,352 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-14T14:41:10,353 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true 2024-11-14T14:41:10,353 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:10,354 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:10,354 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T14:41:10,354 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:10,354 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:10,354 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:10,354 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:10,354 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:10,354 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:10,354 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,40103,1731595261206 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:10,354 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.1731595262836, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:10,354 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-14T14:41:10,355 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=61 2024-11-14T14:41:10,355 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=da1c6afcd1f9,40103,1731595261206, table=hbase:meta, region=1588230740 2024-11-14T14:41:10,355 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as da1c6afcd1f9,40103,1731595261206, state=OPEN 2024-11-14T14:41:10,358 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34057-0x101a73a44750000, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:10,358 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46229-0x101a73a44750003, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:10,358 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40103-0x101a73a44750001, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:10,358 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:10,358 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:10,358 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:10,396 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731595300396 2024-11-14T14:41:10,396 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-14T14:41:10,407 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3613f06851ef4465281b91596c2f4aa4, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,408 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7030563781ca9d5e4d044b06d29712c, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,408 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1c1eeeedb7ad89130474f95f0b19f70, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,408 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8dbd58303faefb2f8b5972b0c3fc3aea, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,408 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e71d8bcd86c479813fc8c2e3a01b917, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,408 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d35b9113ecfdcde0da6d3c51d6891527, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,408 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9cb318690a770d1645dfd64bec1a284b, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,408 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad7ad94ee453836a534ae596990b42b8, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,408 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=135a286d1aafce40676bb4d4370c9daf, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,408 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d14b6f516265d1b90f14cf4519cf2b0f, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,396 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T14:41:10,408 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d7e464b03c5060f7367637b64f2eae0, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,409 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77f677c4c4cfcadb020d2436881cdeaa, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,409 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-14T14:41:10,409 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9f8e87df66c884919328ba09109d819c, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,409 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f0bff83267f1352a4c903f322542cf14, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,409 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=550a3973a29751310a867312ba0b03d1, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,409 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=624639c373b3612dcb71af916020ab1e, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,409 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e1484a67e6b0eaf08cc5e1d6301f8319, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,409 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b2f6569935d4c7faa1470e22f0b79a20, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,409 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=177291d3bf2c8a0b1e9a3689929e9e50, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,409 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71350ab931ae29ff2210bd9d6c3eeac5, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,409 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d725ecb42ad6f19c89d6ddaa229edc8, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,409 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e2ab5762f8d8f31683b4f4994048f3e, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,410 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbc3f9fabaa1bd9fca0801fe6494782e, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,410 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=954bd925c25bfdb6e075a7471bbb1c4d, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,410 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76be481b709a73fdf12e38c88e45d041, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,410 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1316438481dbf11d64fb665f6cdd4eb6, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,410 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-14T14:41:10,410 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731595330410 2024-11-14T14:41:10,410 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731595390410 2024-11-14T14:41:10,410 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 14 msec 2024-11-14T14:41:10,411 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-14T14:41:10,411 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:10,411 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T14:41:10,411 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,40103,1731595261206 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T14:41:10,661 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-14T14:41:10,661 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T14:41:10,661 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T14:41:10,661 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T14:41:10,661 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-14T14:41:10,661 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-14T14:41:10,661 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-14T14:41:10,661 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T14:41:10,662 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-14T14:41:10,664 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-14T14:41:10,664 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T14:41:10,664 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T14:41:10,664 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:10,664 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T14:41:10,664 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T14:41:10,665 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-14T14:41:10,666 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true 2024-11-14T14:41:10,666 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:10,666 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:10,666 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T14:41:10,666 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:10,666 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:10,666 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:10,666 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:10,666 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:10,666 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,40103,1731595261206 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:10,666 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-14T14:41:10,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=61 2024-11-14T14:41:10,669 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=da1c6afcd1f9,40103,1731595261206, table=hbase:meta, region=1588230740 2024-11-14T14:41:10,669 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as da1c6afcd1f9,40103,1731595261206, state=OPEN 2024-11-14T14:41:10,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34057-0x101a73a44750000, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:10,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46229-0x101a73a44750003, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:10,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40103-0x101a73a44750001, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:10,671 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:10,671 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:10,673 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:10,673 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731595300673 2024-11-14T14:41:10,673 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-14T14:41:10,673 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T14:41:10,674 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-14T14:41:10,675 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-14T14:41:10,676 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T14:41:10,676 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T14:41:10,676 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T14:41:10,679 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3613f06851ef4465281b91596c2f4aa4, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,679 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7030563781ca9d5e4d044b06d29712c, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,679 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1c1eeeedb7ad89130474f95f0b19f70, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,679 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8dbd58303faefb2f8b5972b0c3fc3aea, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,679 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e71d8bcd86c479813fc8c2e3a01b917, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,679 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d35b9113ecfdcde0da6d3c51d6891527, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,679 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9cb318690a770d1645dfd64bec1a284b, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,680 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad7ad94ee453836a534ae596990b42b8, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,680 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=135a286d1aafce40676bb4d4370c9daf, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,680 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d14b6f516265d1b90f14cf4519cf2b0f, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,680 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d7e464b03c5060f7367637b64f2eae0, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,680 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77f677c4c4cfcadb020d2436881cdeaa, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,680 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9f8e87df66c884919328ba09109d819c, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,680 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f0bff83267f1352a4c903f322542cf14, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,680 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=550a3973a29751310a867312ba0b03d1, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,681 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=624639c373b3612dcb71af916020ab1e, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,681 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e1484a67e6b0eaf08cc5e1d6301f8319, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,681 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b2f6569935d4c7faa1470e22f0b79a20, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,681 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=177291d3bf2c8a0b1e9a3689929e9e50, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,681 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71350ab931ae29ff2210bd9d6c3eeac5, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,681 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d725ecb42ad6f19c89d6ddaa229edc8, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,681 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e2ab5762f8d8f31683b4f4994048f3e, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,681 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbc3f9fabaa1bd9fca0801fe6494782e, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,681 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=954bd925c25bfdb6e075a7471bbb1c4d, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,682 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76be481b709a73fdf12e38c88e45d041, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,682 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1316438481dbf11d64fb665f6cdd4eb6, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,682 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-14T14:41:10,682 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731595330682 2024-11-14T14:41:10,682 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731595390682 2024-11-14T14:41:10,682 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-14T14:41:10,682 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-14T14:41:10,682 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T14:41:10,682 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T14:41:10,682 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T14:41:10,682 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-14T14:41:10,682 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-14T14:41:10,683 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-14T14:41:10,683 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T14:41:10,683 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34057 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T14:41:10,684 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-14T14:41:10,686 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34057 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T14:41:10,687 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-14T14:41:10,687 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T14:41:10,687 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T14:41:10,687 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T14:41:10,687 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T14:41:10,688 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-14T14:41:10,689 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true 2024-11-14T14:41:10,689 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:10,689 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:10,689 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T14:41:10,689 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:10,689 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:10,689 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:10,689 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:10,689 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-14T14:41:10,698 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=61 2024-11-14T14:41:10,698 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=da1c6afcd1f9,40103,1731595261206, table=hbase:meta, region=1588230740 2024-11-14T14:41:10,701 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as da1c6afcd1f9,40103,1731595261206, state=OPEN 2024-11-14T14:41:10,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46229-0x101a73a44750003, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:10,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40103-0x101a73a44750001, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:10,705 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34057-0x101a73a44750000, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:10,705 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:10,705 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:10,705 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:10,712 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731595300712 2024-11-14T14:41:10,712 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-14T14:41:10,712 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T14:41:10,713 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-14T14:41:10,719 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-14T14:41:10,719 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:10,719 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T14:41:10,719 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T14:41:10,723 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3613f06851ef4465281b91596c2f4aa4, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,723 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7030563781ca9d5e4d044b06d29712c, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,723 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1c1eeeedb7ad89130474f95f0b19f70, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,723 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8dbd58303faefb2f8b5972b0c3fc3aea, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,724 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e71d8bcd86c479813fc8c2e3a01b917, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,724 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d35b9113ecfdcde0da6d3c51d6891527, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,724 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9cb318690a770d1645dfd64bec1a284b, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,724 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad7ad94ee453836a534ae596990b42b8, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,724 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=135a286d1aafce40676bb4d4370c9daf, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,724 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d14b6f516265d1b90f14cf4519cf2b0f, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,724 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d7e464b03c5060f7367637b64f2eae0, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,725 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77f677c4c4cfcadb020d2436881cdeaa, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,725 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9f8e87df66c884919328ba09109d819c, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,725 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f0bff83267f1352a4c903f322542cf14, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,725 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=550a3973a29751310a867312ba0b03d1, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,725 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=624639c373b3612dcb71af916020ab1e, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,725 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e1484a67e6b0eaf08cc5e1d6301f8319, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,725 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b2f6569935d4c7faa1470e22f0b79a20, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,725 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=177291d3bf2c8a0b1e9a3689929e9e50, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,726 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71350ab931ae29ff2210bd9d6c3eeac5, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,726 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d725ecb42ad6f19c89d6ddaa229edc8, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,726 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e2ab5762f8d8f31683b4f4994048f3e, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,726 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbc3f9fabaa1bd9fca0801fe6494782e, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,726 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=954bd925c25bfdb6e075a7471bbb1c4d, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,726 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76be481b709a73fdf12e38c88e45d041, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,726 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1316438481dbf11d64fb665f6cdd4eb6, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,726 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-14T14:41:10,727 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731595330726 2024-11-14T14:41:10,727 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731595390727 2024-11-14T14:41:10,727 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 14 msec 2024-11-14T14:41:10,727 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-14T14:41:10,727 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T14:41:10,727 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T14:41:10,727 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T14:41:10,729 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-14T14:41:10,730 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-14T14:41:10,730 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-14T14:41:10,730 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T14:41:10,732 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-14T14:41:10,734 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-14T14:41:10,734 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T14:41:10,734 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T14:41:10,734 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T14:41:10,735 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-14T14:41:10,735 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true 2024-11-14T14:41:10,736 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:10,736 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:10,736 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:10,736 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:10,736 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:10,736 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-14T14:41:10,761 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=61 2024-11-14T14:41:10,761 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=da1c6afcd1f9,40103,1731595261206, table=hbase:meta, region=1588230740 2024-11-14T14:41:10,761 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as da1c6afcd1f9,40103,1731595261206, state=OPEN 2024-11-14T14:41:10,763 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46229-0x101a73a44750003, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:10,763 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:10,763 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40103-0x101a73a44750001, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:10,763 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34057-0x101a73a44750000, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:10,763 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:10,764 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:10,792 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731595300792 2024-11-14T14:41:10,793 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-14T14:41:10,793 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T14:41:10,794 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-14T14:41:10,795 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-14T14:41:10,795 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:10,795 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T14:41:10,795 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T14:41:10,799 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3613f06851ef4465281b91596c2f4aa4, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,799 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7030563781ca9d5e4d044b06d29712c, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,799 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1c1eeeedb7ad89130474f95f0b19f70, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,799 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8dbd58303faefb2f8b5972b0c3fc3aea, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,799 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e71d8bcd86c479813fc8c2e3a01b917, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,799 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d35b9113ecfdcde0da6d3c51d6891527, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,799 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9cb318690a770d1645dfd64bec1a284b, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,799 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad7ad94ee453836a534ae596990b42b8, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,799 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=135a286d1aafce40676bb4d4370c9daf, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,800 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d14b6f516265d1b90f14cf4519cf2b0f, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,800 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d7e464b03c5060f7367637b64f2eae0, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,800 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77f677c4c4cfcadb020d2436881cdeaa, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,800 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9f8e87df66c884919328ba09109d819c, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,800 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f0bff83267f1352a4c903f322542cf14, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,800 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=550a3973a29751310a867312ba0b03d1, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,800 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=624639c373b3612dcb71af916020ab1e, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,800 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e1484a67e6b0eaf08cc5e1d6301f8319, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,800 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b2f6569935d4c7faa1470e22f0b79a20, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,800 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=177291d3bf2c8a0b1e9a3689929e9e50, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,800 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71350ab931ae29ff2210bd9d6c3eeac5, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,801 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d725ecb42ad6f19c89d6ddaa229edc8, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,801 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e2ab5762f8d8f31683b4f4994048f3e, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,801 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbc3f9fabaa1bd9fca0801fe6494782e, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,801 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=954bd925c25bfdb6e075a7471bbb1c4d, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,801 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76be481b709a73fdf12e38c88e45d041, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,801 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1316438481dbf11d64fb665f6cdd4eb6, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,801 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-14T14:41:10,801 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731595330801 2024-11-14T14:41:10,801 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731595390801 2024-11-14T14:41:10,801 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-14T14:41:10,801 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-14T14:41:10,801 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T14:41:10,801 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T14:41:10,801 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T14:41:10,802 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-14T14:41:10,802 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-14T14:41:10,802 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-14T14:41:10,802 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T14:41:10,803 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-14T14:41:10,805 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-14T14:41:10,805 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T14:41:10,805 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T14:41:10,805 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-14T14:41:10,806 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true 2024-11-14T14:41:10,806 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:10,806 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:10,806 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure da1c6afcd1f9%2C46729%2C1731595261321.meta.1731595263293.meta, worker=da1c6afcd1f9,46229,1731595261363 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T14:41:10,806 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-14T14:41:10,824 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=61 2024-11-14T14:41:10,824 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=da1c6afcd1f9,40103,1731595261206, table=hbase:meta, region=1588230740 2024-11-14T14:41:10,825 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as da1c6afcd1f9,40103,1731595261206, state=OPEN 2024-11-14T14:41:10,827 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46229-0x101a73a44750003, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:10,827 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34057-0x101a73a44750000, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:10,827 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40103-0x101a73a44750001, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:10,827 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:10,827 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:10,827 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:10,848 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731595300848 2024-11-14T14:41:10,848 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-14T14:41:10,857 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3613f06851ef4465281b91596c2f4aa4, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,857 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7030563781ca9d5e4d044b06d29712c, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,857 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1c1eeeedb7ad89130474f95f0b19f70, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,857 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8dbd58303faefb2f8b5972b0c3fc3aea, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,857 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e71d8bcd86c479813fc8c2e3a01b917, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,857 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d35b9113ecfdcde0da6d3c51d6891527, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,858 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9cb318690a770d1645dfd64bec1a284b, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,858 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad7ad94ee453836a534ae596990b42b8, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,858 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=135a286d1aafce40676bb4d4370c9daf, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,858 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d14b6f516265d1b90f14cf4519cf2b0f, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,858 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d7e464b03c5060f7367637b64f2eae0, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,858 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77f677c4c4cfcadb020d2436881cdeaa, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,858 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9f8e87df66c884919328ba09109d819c, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,858 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f0bff83267f1352a4c903f322542cf14, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,858 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=550a3973a29751310a867312ba0b03d1, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,858 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=624639c373b3612dcb71af916020ab1e, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,858 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e1484a67e6b0eaf08cc5e1d6301f8319, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,858 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b2f6569935d4c7faa1470e22f0b79a20, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,859 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=177291d3bf2c8a0b1e9a3689929e9e50, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,859 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71350ab931ae29ff2210bd9d6c3eeac5, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,859 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d725ecb42ad6f19c89d6ddaa229edc8, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,859 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e2ab5762f8d8f31683b4f4994048f3e, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,859 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbc3f9fabaa1bd9fca0801fe6494782e, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,859 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=954bd925c25bfdb6e075a7471bbb1c4d, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:10,859 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76be481b709a73fdf12e38c88e45d041, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,859 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1316438481dbf11d64fb665f6cdd4eb6, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:10,859 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-14T14:41:10,859 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731595330859 2024-11-14T14:41:10,859 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731595390859 2024-11-14T14:41:10,859 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 11 msec 2024-11-14T14:41:10,860 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T14:41:10,861 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-14T14:41:10,863 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-14T14:41:10,863 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true 2024-11-14T14:41:10,863 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T14:41:10,863 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1604) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T14:41:10,926 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-14T14:41:10,927 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-14T14:41:10,929 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test 2024-11-14T14:41:10,929 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test Metrics about Tables on a single HBase RegionServer 2024-11-14T14:41:10,931 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-14T14:41:10,931 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-14T14:41:10,931 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-14T14:41:10,931 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-14T14:41:11,110 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-14T14:41:11,110 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T14:41:11,110 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T14:41:11,110 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T14:41:11,110 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-14T14:41:11,110 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-14T14:41:11,110 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-14T14:41:11,111 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T14:41:11,116 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 5 msec 2024-11-14T14:41:11,118 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-14T14:41:11,118 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T14:41:11,119 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-14T14:41:11,119 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true 2024-11-14T14:41:11,119 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-14T14:41:11,120 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-14T14:41:11,120 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=61 2024-11-14T14:41:11,120 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=da1c6afcd1f9,40103,1731595261206, table=hbase:meta, region=1588230740 2024-11-14T14:41:11,121 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as da1c6afcd1f9,40103,1731595261206, state=OPEN 2024-11-14T14:41:11,123 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40103-0x101a73a44750001, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:11,123 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46229-0x101a73a44750003, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:11,123 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:11,123 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:11,123 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34057-0x101a73a44750000, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:41:11,123 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:41:11,128 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731595301128 2024-11-14T14:41:11,128 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-14T14:41:11,128 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T14:41:11,130 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-14T14:41:11,135 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3613f06851ef4465281b91596c2f4aa4, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:11,135 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7030563781ca9d5e4d044b06d29712c, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:11,135 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1c1eeeedb7ad89130474f95f0b19f70, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:11,135 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8dbd58303faefb2f8b5972b0c3fc3aea, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:11,135 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e71d8bcd86c479813fc8c2e3a01b917, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:11,135 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d35b9113ecfdcde0da6d3c51d6891527, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:11,135 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9cb318690a770d1645dfd64bec1a284b, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:11,135 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad7ad94ee453836a534ae596990b42b8, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:11,135 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=135a286d1aafce40676bb4d4370c9daf, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:11,135 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d14b6f516265d1b90f14cf4519cf2b0f, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:11,136 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d7e464b03c5060f7367637b64f2eae0, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:11,136 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77f677c4c4cfcadb020d2436881cdeaa, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:11,136 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9f8e87df66c884919328ba09109d819c, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:11,136 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f0bff83267f1352a4c903f322542cf14, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:11,136 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=550a3973a29751310a867312ba0b03d1, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:11,136 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=624639c373b3612dcb71af916020ab1e, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:11,136 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e1484a67e6b0eaf08cc5e1d6301f8319, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:11,136 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=57, state=ROLLEDBACK, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 2e2ab5762f8d8f31683b4f4994048f3e, server=da1c6afcd1f9,40103,1731595261206:java.lang.RuntimeException: inject code bug; ServerCrashProcedure da1c6afcd1f9,46729,1731595261321, splitWal=true, meta=true exec-time=4.7070 sec 2024-11-14T14:41:11,136 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b2f6569935d4c7faa1470e22f0b79a20, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:11,136 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=177291d3bf2c8a0b1e9a3689929e9e50, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:11,136 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71350ab931ae29ff2210bd9d6c3eeac5, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:11,136 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d725ecb42ad6f19c89d6ddaa229edc8, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:11,136 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e2ab5762f8d8f31683b4f4994048f3e, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:11,136 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbc3f9fabaa1bd9fca0801fe6494782e, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:11,137 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=954bd925c25bfdb6e075a7471bbb1c4d, regionState=OPEN, lastHost=da1c6afcd1f9,40103,1731595261206, regionLocation=da1c6afcd1f9,40103,1731595261206, openSeqNum=2 2024-11-14T14:41:11,137 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=76be481b709a73fdf12e38c88e45d041, regionState=OPENING, lastHost=da1c6afcd1f9,46729,1731595261321, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:11,137 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1316438481dbf11d64fb665f6cdd4eb6, regionState=OPEN, lastHost=da1c6afcd1f9,46229,1731595261363, regionLocation=da1c6afcd1f9,46229,1731595261363, openSeqNum=2 2024-11-14T14:41:11,137 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-14T14:41:11,137 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731595331137 2024-11-14T14:41:11,137 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731595391137 2024-11-14T14:41:11,137 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-14T14:41:11,162 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.assignment.TestRollbackSCP#testFailAndRollback Thread=348 (was 382), OpenFileDescriptor=618 (was 641), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=445 (was 414) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8145 (was 8233) 2024-11-14T14:41:11,162 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-14T14:41:11,162 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T14:41:11,162 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T14:41:11,162 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:41:11,163 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:41:11,163 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T14:41:11,163 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-14T14:41:11,163 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=291071084, stopped=false 2024-11-14T14:41:11,164 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=da1c6afcd1f9,34057,1731595260501 2024-11-14T14:41:11,166 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40103-0x101a73a44750001, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T14:41:11,166 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46229-0x101a73a44750003, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T14:41:11,166 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40103-0x101a73a44750001, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:41:11,166 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46229-0x101a73a44750003, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:41:11,166 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T14:41:11,166 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34057-0x101a73a44750000, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T14:41:11,166 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34057-0x101a73a44750000, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:41:11,166 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T14:41:11,166 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T14:41:11,166 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:41:11,167 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'da1c6afcd1f9,40103,1731595261206' ***** 2024-11-14T14:41:11,167 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-14T14:41:11,167 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'da1c6afcd1f9,46229,1731595261363' ***** 2024-11-14T14:41:11,167 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-14T14:41:11,167 INFO [RS:0;da1c6afcd1f9:40103 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-14T14:41:11,167 INFO [RS:0;da1c6afcd1f9:40103 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-14T14:41:11,167 INFO [RS:0;da1c6afcd1f9:40103 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-14T14:41:11,168 INFO [RS:0;da1c6afcd1f9:40103 {}] regionserver.HRegionServer(3091): Received CLOSE for d14b6f516265d1b90f14cf4519cf2b0f 2024-11-14T14:41:11,168 INFO [RS:2;da1c6afcd1f9:46229 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-14T14:41:11,168 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-14T14:41:11,168 INFO [RS:2;da1c6afcd1f9:46229 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-14T14:41:11,168 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-14T14:41:11,168 INFO [RS:2;da1c6afcd1f9:46229 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-14T14:41:11,169 INFO [RS:2;da1c6afcd1f9:46229 {}] regionserver.HRegionServer(3091): Received CLOSE for 1d7e464b03c5060f7367637b64f2eae0 2024-11-14T14:41:11,169 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:34057-0x101a73a44750000, quorum=127.0.0.1:60650, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T14:41:11,169 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40103-0x101a73a44750001, quorum=127.0.0.1:60650, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T14:41:11,169 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46229-0x101a73a44750003, quorum=127.0.0.1:60650, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T14:41:11,170 INFO [regionserver/da1c6afcd1f9:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T14:41:11,180 INFO [regionserver/da1c6afcd1f9:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T14:41:11,180 INFO [RS:0;da1c6afcd1f9:40103 {}] regionserver.HRegionServer(3091): Received CLOSE for f0bff83267f1352a4c903f322542cf14 2024-11-14T14:41:11,180 INFO [RS:0;da1c6afcd1f9:40103 {}] regionserver.HRegionServer(3091): Received CLOSE for b7030563781ca9d5e4d044b06d29712c 2024-11-14T14:41:11,180 INFO [RS:0;da1c6afcd1f9:40103 {}] regionserver.HRegionServer(3091): Received CLOSE for b2f6569935d4c7faa1470e22f0b79a20 2024-11-14T14:41:11,180 INFO [RS:0;da1c6afcd1f9:40103 {}] regionserver.HRegionServer(3091): Received CLOSE for e1484a67e6b0eaf08cc5e1d6301f8319 2024-11-14T14:41:11,180 INFO [RS:0;da1c6afcd1f9:40103 {}] regionserver.HRegionServer(3091): Received CLOSE for 8dbd58303faefb2f8b5972b0c3fc3aea 2024-11-14T14:41:11,180 INFO [RS:0;da1c6afcd1f9:40103 {}] regionserver.HRegionServer(3091): Received CLOSE for 3613f06851ef4465281b91596c2f4aa4 2024-11-14T14:41:11,180 INFO [RS:0;da1c6afcd1f9:40103 {}] regionserver.HRegionServer(3091): Received CLOSE for ad7ad94ee453836a534ae596990b42b8 2024-11-14T14:41:11,180 INFO [RS:0;da1c6afcd1f9:40103 {}] regionserver.HRegionServer(3091): Received CLOSE for 6d725ecb42ad6f19c89d6ddaa229edc8 2024-11-14T14:41:11,180 INFO [RS:0;da1c6afcd1f9:40103 {}] regionserver.HRegionServer(3091): Received CLOSE for 2e2ab5762f8d8f31683b4f4994048f3e 2024-11-14T14:41:11,180 INFO [RS:0;da1c6afcd1f9:40103 {}] regionserver.HRegionServer(3091): Received CLOSE for 550a3973a29751310a867312ba0b03d1 2024-11-14T14:41:11,180 INFO [RS:0;da1c6afcd1f9:40103 {}] regionserver.HRegionServer(3091): Received CLOSE for 954bd925c25bfdb6e075a7471bbb1c4d 2024-11-14T14:41:11,180 INFO [RS:0;da1c6afcd1f9:40103 {}] regionserver.HRegionServer(3091): Received CLOSE for 135a286d1aafce40676bb4d4370c9daf 2024-11-14T14:41:11,180 INFO [RS:0;da1c6afcd1f9:40103 {}] regionserver.HRegionServer(959): stopping server da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:11,181 INFO [RS:0;da1c6afcd1f9:40103 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T14:41:11,181 INFO [RS:0;da1c6afcd1f9:40103 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;da1c6afcd1f9:40103. 2024-11-14T14:41:11,181 DEBUG [RS:0;da1c6afcd1f9:40103 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T14:41:11,181 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d14b6f516265d1b90f14cf4519cf2b0f, disabling compactions & flushes 2024-11-14T14:41:11,181 DEBUG [RS:0;da1c6afcd1f9:40103 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:41:11,181 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,jjj,1731595263719.d14b6f516265d1b90f14cf4519cf2b0f. 2024-11-14T14:41:11,181 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1731595263719.d14b6f516265d1b90f14cf4519cf2b0f. 2024-11-14T14:41:11,181 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1731595263719.d14b6f516265d1b90f14cf4519cf2b0f. after waiting 0 ms 2024-11-14T14:41:11,181 INFO [RS:0;da1c6afcd1f9:40103 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-14T14:41:11,181 INFO [RS:0;da1c6afcd1f9:40103 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-14T14:41:11,181 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1731595263719.d14b6f516265d1b90f14cf4519cf2b0f. 2024-11-14T14:41:11,181 INFO [RS:0;da1c6afcd1f9:40103 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-14T14:41:11,181 INFO [RS:0;da1c6afcd1f9:40103 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-14T14:41:11,216 INFO [RS:2;da1c6afcd1f9:46229 {}] regionserver.HRegionServer(3091): Received CLOSE for c1c1eeeedb7ad89130474f95f0b19f70 2024-11-14T14:41:11,216 INFO [RS:2;da1c6afcd1f9:46229 {}] regionserver.HRegionServer(3091): Received CLOSE for 76be481b709a73fdf12e38c88e45d041 2024-11-14T14:41:11,216 INFO [RS:2;da1c6afcd1f9:46229 {}] regionserver.HRegionServer(3091): Received CLOSE for dbc3f9fabaa1bd9fca0801fe6494782e 2024-11-14T14:41:11,216 INFO [RS:2;da1c6afcd1f9:46229 {}] regionserver.HRegionServer(3091): Received CLOSE for 624639c373b3612dcb71af916020ab1e 2024-11-14T14:41:11,216 INFO [RS:2;da1c6afcd1f9:46229 {}] regionserver.HRegionServer(3091): Received CLOSE for d35b9113ecfdcde0da6d3c51d6891527 2024-11-14T14:41:11,216 INFO [RS:2;da1c6afcd1f9:46229 {}] regionserver.HRegionServer(3091): Received CLOSE for 5e71d8bcd86c479813fc8c2e3a01b917 2024-11-14T14:41:11,216 INFO [RS:2;da1c6afcd1f9:46229 {}] regionserver.HRegionServer(3091): Received CLOSE for 9cb318690a770d1645dfd64bec1a284b 2024-11-14T14:41:11,217 INFO [RS:2;da1c6afcd1f9:46229 {}] regionserver.HRegionServer(3091): Received CLOSE for 1316438481dbf11d64fb665f6cdd4eb6 2024-11-14T14:41:11,217 INFO [RS:2;da1c6afcd1f9:46229 {}] regionserver.HRegionServer(3091): Received CLOSE for 9f8e87df66c884919328ba09109d819c 2024-11-14T14:41:11,217 INFO [RS:2;da1c6afcd1f9:46229 {}] regionserver.HRegionServer(3091): Received CLOSE for 71350ab931ae29ff2210bd9d6c3eeac5 2024-11-14T14:41:11,217 INFO [RS:2;da1c6afcd1f9:46229 {}] regionserver.HRegionServer(3091): Received CLOSE for 77f677c4c4cfcadb020d2436881cdeaa 2024-11-14T14:41:11,217 INFO [RS:2;da1c6afcd1f9:46229 {}] regionserver.HRegionServer(3091): Received CLOSE for 177291d3bf2c8a0b1e9a3689929e9e50 2024-11-14T14:41:11,217 INFO [RS:2;da1c6afcd1f9:46229 {}] regionserver.HRegionServer(959): stopping server da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:11,217 INFO [RS:2;da1c6afcd1f9:46229 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T14:41:11,217 INFO [RS:2;da1c6afcd1f9:46229 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;da1c6afcd1f9:46229. 2024-11-14T14:41:11,217 DEBUG [RS:2;da1c6afcd1f9:46229 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T14:41:11,217 DEBUG [RS:2;da1c6afcd1f9:46229 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:41:11,217 INFO [RS:2;da1c6afcd1f9:46229 {}] regionserver.HRegionServer(1321): Waiting on 13 regions to close 2024-11-14T14:41:11,218 DEBUG [RS:2;da1c6afcd1f9:46229 {}] regionserver.HRegionServer(1351): Waiting on 1316438481dbf11d64fb665f6cdd4eb6, 177291d3bf2c8a0b1e9a3689929e9e50, 1d7e464b03c5060f7367637b64f2eae0, 5e71d8bcd86c479813fc8c2e3a01b917, 624639c373b3612dcb71af916020ab1e, 71350ab931ae29ff2210bd9d6c3eeac5, 76be481b709a73fdf12e38c88e45d041, 77f677c4c4cfcadb020d2436881cdeaa, 9cb318690a770d1645dfd64bec1a284b, 9f8e87df66c884919328ba09109d819c, c1c1eeeedb7ad89130474f95f0b19f70, d35b9113ecfdcde0da6d3c51d6891527, dbc3f9fabaa1bd9fca0801fe6494782e 2024-11-14T14:41:11,218 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1d7e464b03c5060f7367637b64f2eae0, disabling compactions & flushes 2024-11-14T14:41:11,218 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,kkk,1731595263719.1d7e464b03c5060f7367637b64f2eae0. 2024-11-14T14:41:11,218 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1731595263719.1d7e464b03c5060f7367637b64f2eae0. 2024-11-14T14:41:11,218 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1731595263719.1d7e464b03c5060f7367637b64f2eae0. after waiting 0 ms 2024-11-14T14:41:11,218 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1731595263719.1d7e464b03c5060f7367637b64f2eae0. 2024-11-14T14:41:11,228 INFO [RS:0;da1c6afcd1f9:40103 {}] regionserver.HRegionServer(1321): Waiting on 14 regions to close 2024-11-14T14:41:11,228 DEBUG [RS:0;da1c6afcd1f9:40103 {}] regionserver.HRegionServer(1351): Waiting on 135a286d1aafce40676bb4d4370c9daf, 1588230740, 2e2ab5762f8d8f31683b4f4994048f3e, 3613f06851ef4465281b91596c2f4aa4, 550a3973a29751310a867312ba0b03d1, 6d725ecb42ad6f19c89d6ddaa229edc8, 8dbd58303faefb2f8b5972b0c3fc3aea, 954bd925c25bfdb6e075a7471bbb1c4d, ad7ad94ee453836a534ae596990b42b8, b2f6569935d4c7faa1470e22f0b79a20, b7030563781ca9d5e4d044b06d29712c, d14b6f516265d1b90f14cf4519cf2b0f, e1484a67e6b0eaf08cc5e1d6301f8319, f0bff83267f1352a4c903f322542cf14 2024-11-14T14:41:11,229 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T14:41:11,229 INFO [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T14:41:11,230 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T14:41:11,230 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T14:41:11,230 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T14:41:11,231 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/d14b6f516265d1b90f14cf4519cf2b0f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T14:41:11,236 INFO [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.66 KB heapSize=5.44 KB 2024-11-14T14:41:11,241 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,jjj,1731595263719.d14b6f516265d1b90f14cf4519cf2b0f. 2024-11-14T14:41:11,242 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d14b6f516265d1b90f14cf4519cf2b0f: Waiting for close lock at 1731595271180Running coprocessor pre-close hooks at 1731595271181 (+1 ms)Disabling compacts and flushes for region at 1731595271181Disabling writes for close at 1731595271181Writing region close event to WAL at 1731595271202 (+21 ms)Running coprocessor post-close hooks at 1731595271236 (+34 ms)Closed at 1731595271241 (+5 ms) 2024-11-14T14:41:11,242 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,jjj,1731595263719.d14b6f516265d1b90f14cf4519cf2b0f. 2024-11-14T14:41:11,242 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f0bff83267f1352a4c903f322542cf14, disabling compactions & flushes 2024-11-14T14:41:11,242 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,nnn,1731595263719.f0bff83267f1352a4c903f322542cf14. 2024-11-14T14:41:11,242 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1731595263719.f0bff83267f1352a4c903f322542cf14. 2024-11-14T14:41:11,242 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1731595263719.f0bff83267f1352a4c903f322542cf14. after waiting 0 ms 2024-11-14T14:41:11,242 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1731595263719.f0bff83267f1352a4c903f322542cf14. 2024-11-14T14:41:11,269 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/hbase/meta/1588230740/.tmp/info/df8f915648a84e55aad43e856e5d4423 is 138, key is test,bbb,1731595263719.b7030563781ca9d5e4d044b06d29712c./info:regioninfo/1731595268387/Put/seqid=0 2024-11-14T14:41:11,271 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/1d7e464b03c5060f7367637b64f2eae0/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T14:41:11,272 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,kkk,1731595263719.1d7e464b03c5060f7367637b64f2eae0. 2024-11-14T14:41:11,273 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1d7e464b03c5060f7367637b64f2eae0: Waiting for close lock at 1731595271218Running coprocessor pre-close hooks at 1731595271218Disabling compacts and flushes for region at 1731595271218Disabling writes for close at 1731595271218Writing region close event to WAL at 1731595271244 (+26 ms)Running coprocessor post-close hooks at 1731595271272 (+28 ms)Closed at 1731595271272 2024-11-14T14:41:11,273 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,kkk,1731595263719.1d7e464b03c5060f7367637b64f2eae0. 2024-11-14T14:41:11,273 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c1c1eeeedb7ad89130474f95f0b19f70, disabling compactions & flushes 2024-11-14T14:41:11,273 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ccc,1731595263719.c1c1eeeedb7ad89130474f95f0b19f70. 2024-11-14T14:41:11,273 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1731595263719.c1c1eeeedb7ad89130474f95f0b19f70. 2024-11-14T14:41:11,273 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1731595263719.c1c1eeeedb7ad89130474f95f0b19f70. after waiting 0 ms 2024-11-14T14:41:11,273 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1731595263719.c1c1eeeedb7ad89130474f95f0b19f70. 2024-11-14T14:41:11,294 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/f0bff83267f1352a4c903f322542cf14/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T14:41:11,296 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,nnn,1731595263719.f0bff83267f1352a4c903f322542cf14. 2024-11-14T14:41:11,296 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f0bff83267f1352a4c903f322542cf14: Waiting for close lock at 1731595271242Running coprocessor pre-close hooks at 1731595271242Disabling compacts and flushes for region at 1731595271242Disabling writes for close at 1731595271242Writing region close event to WAL at 1731595271264 (+22 ms)Running coprocessor post-close hooks at 1731595271295 (+31 ms)Closed at 1731595271296 (+1 ms) 2024-11-14T14:41:11,296 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,nnn,1731595263719.f0bff83267f1352a4c903f322542cf14. 2024-11-14T14:41:11,296 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b7030563781ca9d5e4d044b06d29712c, disabling compactions & flushes 2024-11-14T14:41:11,296 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,bbb,1731595263719.b7030563781ca9d5e4d044b06d29712c. 2024-11-14T14:41:11,296 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1731595263719.b7030563781ca9d5e4d044b06d29712c. 2024-11-14T14:41:11,296 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1731595263719.b7030563781ca9d5e4d044b06d29712c. after waiting 0 ms 2024-11-14T14:41:11,297 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1731595263719.b7030563781ca9d5e4d044b06d29712c. 2024-11-14T14:41:11,304 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/c1c1eeeedb7ad89130474f95f0b19f70/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T14:41:11,305 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ccc,1731595263719.c1c1eeeedb7ad89130474f95f0b19f70. 2024-11-14T14:41:11,305 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c1c1eeeedb7ad89130474f95f0b19f70: Waiting for close lock at 1731595271273Running coprocessor pre-close hooks at 1731595271273Disabling compacts and flushes for region at 1731595271273Disabling writes for close at 1731595271273Writing region close event to WAL at 1731595271284 (+11 ms)Running coprocessor post-close hooks at 1731595271305 (+21 ms)Closed at 1731595271305 2024-11-14T14:41:11,305 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ccc,1731595263719.c1c1eeeedb7ad89130474f95f0b19f70. 2024-11-14T14:41:11,305 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 76be481b709a73fdf12e38c88e45d041, disabling compactions & flushes 2024-11-14T14:41:11,306 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,yyy,1731595263719.76be481b709a73fdf12e38c88e45d041. 2024-11-14T14:41:11,306 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1731595263719.76be481b709a73fdf12e38c88e45d041. 2024-11-14T14:41:11,306 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1731595263719.76be481b709a73fdf12e38c88e45d041. after waiting 0 ms 2024-11-14T14:41:11,306 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1731595263719.76be481b709a73fdf12e38c88e45d041. 2024-11-14T14:41:11,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45021 is added to blk_1073741869_1045 (size=8195) 2024-11-14T14:41:11,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45825 is added to blk_1073741869_1045 (size=8195) 2024-11-14T14:41:11,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36849 is added to blk_1073741869_1045 (size=8195) 2024-11-14T14:41:11,342 INFO [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.66 KB at sequenceid=71 (bloomFilter=true), to=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/hbase/meta/1588230740/.tmp/info/df8f915648a84e55aad43e856e5d4423 2024-11-14T14:41:11,348 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/b7030563781ca9d5e4d044b06d29712c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T14:41:11,348 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/76be481b709a73fdf12e38c88e45d041/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T14:41:11,349 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,bbb,1731595263719.b7030563781ca9d5e4d044b06d29712c. 2024-11-14T14:41:11,350 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,yyy,1731595263719.76be481b709a73fdf12e38c88e45d041. 2024-11-14T14:41:11,350 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 76be481b709a73fdf12e38c88e45d041: Waiting for close lock at 1731595271305Running coprocessor pre-close hooks at 1731595271305Disabling compacts and flushes for region at 1731595271305Disabling writes for close at 1731595271306 (+1 ms)Writing region close event to WAL at 1731595271324 (+18 ms)Running coprocessor post-close hooks at 1731595271349 (+25 ms)Closed at 1731595271349 2024-11-14T14:41:11,350 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b7030563781ca9d5e4d044b06d29712c: Waiting for close lock at 1731595271296Running coprocessor pre-close hooks at 1731595271296Disabling compacts and flushes for region at 1731595271296Disabling writes for close at 1731595271296Writing region close event to WAL at 1731595271312 (+16 ms)Running coprocessor post-close hooks at 1731595271349 (+37 ms)Closed at 1731595271349 2024-11-14T14:41:11,350 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,bbb,1731595263719.b7030563781ca9d5e4d044b06d29712c. 2024-11-14T14:41:11,350 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,yyy,1731595263719.76be481b709a73fdf12e38c88e45d041. 2024-11-14T14:41:11,350 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b2f6569935d4c7faa1470e22f0b79a20, disabling compactions & flushes 2024-11-14T14:41:11,350 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing dbc3f9fabaa1bd9fca0801fe6494782e, disabling compactions & flushes 2024-11-14T14:41:11,350 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,rrr,1731595263719.b2f6569935d4c7faa1470e22f0b79a20. 2024-11-14T14:41:11,350 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,www,1731595263719.dbc3f9fabaa1bd9fca0801fe6494782e. 2024-11-14T14:41:11,350 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1731595263719.dbc3f9fabaa1bd9fca0801fe6494782e. 2024-11-14T14:41:11,350 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1731595263719.b2f6569935d4c7faa1470e22f0b79a20. 2024-11-14T14:41:11,350 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1731595263719.b2f6569935d4c7faa1470e22f0b79a20. after waiting 0 ms 2024-11-14T14:41:11,350 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,www,1731595263719.dbc3f9fabaa1bd9fca0801fe6494782e. after waiting 0 ms 2024-11-14T14:41:11,350 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,www,1731595263719.dbc3f9fabaa1bd9fca0801fe6494782e. 2024-11-14T14:41:11,350 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1731595263719.b2f6569935d4c7faa1470e22f0b79a20. 2024-11-14T14:41:11,354 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/hbase/meta/1588230740/.tmp/info/df8f915648a84e55aad43e856e5d4423 as hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/hbase/meta/1588230740/info/df8f915648a84e55aad43e856e5d4423 2024-11-14T14:41:11,361 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/dbc3f9fabaa1bd9fca0801fe6494782e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T14:41:11,362 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,www,1731595263719.dbc3f9fabaa1bd9fca0801fe6494782e. 2024-11-14T14:41:11,362 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for dbc3f9fabaa1bd9fca0801fe6494782e: Waiting for close lock at 1731595271350Running coprocessor pre-close hooks at 1731595271350Disabling compacts and flushes for region at 1731595271350Disabling writes for close at 1731595271350Writing region close event to WAL at 1731595271352 (+2 ms)Running coprocessor post-close hooks at 1731595271362 (+10 ms)Closed at 1731595271362 2024-11-14T14:41:11,362 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,www,1731595263719.dbc3f9fabaa1bd9fca0801fe6494782e. 2024-11-14T14:41:11,362 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 624639c373b3612dcb71af916020ab1e, disabling compactions & flushes 2024-11-14T14:41:11,362 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ppp,1731595263719.624639c373b3612dcb71af916020ab1e. 2024-11-14T14:41:11,363 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1731595263719.624639c373b3612dcb71af916020ab1e. 2024-11-14T14:41:11,363 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1731595263719.624639c373b3612dcb71af916020ab1e. after waiting 0 ms 2024-11-14T14:41:11,363 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1731595263719.624639c373b3612dcb71af916020ab1e. 2024-11-14T14:41:11,384 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/b2f6569935d4c7faa1470e22f0b79a20/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T14:41:11,386 INFO [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/hbase/meta/1588230740/info/df8f915648a84e55aad43e856e5d4423, entries=24, sequenceid=71, filesize=8.0 K 2024-11-14T14:41:11,387 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,rrr,1731595263719.b2f6569935d4c7faa1470e22f0b79a20. 2024-11-14T14:41:11,387 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b2f6569935d4c7faa1470e22f0b79a20: Waiting for close lock at 1731595271350Running coprocessor pre-close hooks at 1731595271350Disabling compacts and flushes for region at 1731595271350Disabling writes for close at 1731595271350Writing region close event to WAL at 1731595271353 (+3 ms)Running coprocessor post-close hooks at 1731595271387 (+34 ms)Closed at 1731595271387 2024-11-14T14:41:11,387 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,rrr,1731595263719.b2f6569935d4c7faa1470e22f0b79a20. 2024-11-14T14:41:11,387 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e1484a67e6b0eaf08cc5e1d6301f8319, disabling compactions & flushes 2024-11-14T14:41:11,387 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,qqq,1731595263719.e1484a67e6b0eaf08cc5e1d6301f8319. 2024-11-14T14:41:11,388 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1731595263719.e1484a67e6b0eaf08cc5e1d6301f8319. 2024-11-14T14:41:11,388 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1731595263719.e1484a67e6b0eaf08cc5e1d6301f8319. after waiting 0 ms 2024-11-14T14:41:11,388 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1731595263719.e1484a67e6b0eaf08cc5e1d6301f8319. 2024-11-14T14:41:11,388 INFO [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.66 KB/2720, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 158ms, sequenceid=71, compaction requested=false 2024-11-14T14:41:11,409 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/624639c373b3612dcb71af916020ab1e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T14:41:11,410 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ppp,1731595263719.624639c373b3612dcb71af916020ab1e. 2024-11-14T14:41:11,410 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 624639c373b3612dcb71af916020ab1e: Waiting for close lock at 1731595271362Running coprocessor pre-close hooks at 1731595271362Disabling compacts and flushes for region at 1731595271362Disabling writes for close at 1731595271363 (+1 ms)Writing region close event to WAL at 1731595271392 (+29 ms)Running coprocessor post-close hooks at 1731595271410 (+18 ms)Closed at 1731595271410 2024-11-14T14:41:11,410 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ppp,1731595263719.624639c373b3612dcb71af916020ab1e. 2024-11-14T14:41:11,411 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d35b9113ecfdcde0da6d3c51d6891527, disabling compactions & flushes 2024-11-14T14:41:11,411 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,fff,1731595263719.d35b9113ecfdcde0da6d3c51d6891527. 2024-11-14T14:41:11,411 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1731595263719.d35b9113ecfdcde0da6d3c51d6891527. 2024-11-14T14:41:11,411 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,fff,1731595263719.d35b9113ecfdcde0da6d3c51d6891527. after waiting 0 ms 2024-11-14T14:41:11,411 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,fff,1731595263719.d35b9113ecfdcde0da6d3c51d6891527. 2024-11-14T14:41:11,418 DEBUG [RS:2;da1c6afcd1f9:46229 {}] regionserver.HRegionServer(1351): Waiting on 1316438481dbf11d64fb665f6cdd4eb6, 177291d3bf2c8a0b1e9a3689929e9e50, 5e71d8bcd86c479813fc8c2e3a01b917, 71350ab931ae29ff2210bd9d6c3eeac5, 77f677c4c4cfcadb020d2436881cdeaa, 9cb318690a770d1645dfd64bec1a284b, 9f8e87df66c884919328ba09109d819c, d35b9113ecfdcde0da6d3c51d6891527 2024-11-14T14:41:11,425 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/e1484a67e6b0eaf08cc5e1d6301f8319/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T14:41:11,426 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,qqq,1731595263719.e1484a67e6b0eaf08cc5e1d6301f8319. 2024-11-14T14:41:11,426 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e1484a67e6b0eaf08cc5e1d6301f8319: Waiting for close lock at 1731595271387Running coprocessor pre-close hooks at 1731595271387Disabling compacts and flushes for region at 1731595271387Disabling writes for close at 1731595271388 (+1 ms)Writing region close event to WAL at 1731595271397 (+9 ms)Running coprocessor post-close hooks at 1731595271426 (+29 ms)Closed at 1731595271426 2024-11-14T14:41:11,426 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,qqq,1731595263719.e1484a67e6b0eaf08cc5e1d6301f8319. 2024-11-14T14:41:11,427 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8dbd58303faefb2f8b5972b0c3fc3aea, disabling compactions & flushes 2024-11-14T14:41:11,427 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ddd,1731595263719.8dbd58303faefb2f8b5972b0c3fc3aea. 2024-11-14T14:41:11,427 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1731595263719.8dbd58303faefb2f8b5972b0c3fc3aea. 2024-11-14T14:41:11,427 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1731595263719.8dbd58303faefb2f8b5972b0c3fc3aea. after waiting 0 ms 2024-11-14T14:41:11,427 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1731595263719.8dbd58303faefb2f8b5972b0c3fc3aea. 2024-11-14T14:41:11,428 DEBUG [RS:0;da1c6afcd1f9:40103 {}] regionserver.HRegionServer(1351): Waiting on 135a286d1aafce40676bb4d4370c9daf, 1588230740, 2e2ab5762f8d8f31683b4f4994048f3e, 3613f06851ef4465281b91596c2f4aa4, 550a3973a29751310a867312ba0b03d1, 6d725ecb42ad6f19c89d6ddaa229edc8, 8dbd58303faefb2f8b5972b0c3fc3aea, 954bd925c25bfdb6e075a7471bbb1c4d, ad7ad94ee453836a534ae596990b42b8 2024-11-14T14:41:11,446 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/d35b9113ecfdcde0da6d3c51d6891527/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T14:41:11,452 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,fff,1731595263719.d35b9113ecfdcde0da6d3c51d6891527. 2024-11-14T14:41:11,452 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d35b9113ecfdcde0da6d3c51d6891527: Waiting for close lock at 1731595271410Running coprocessor pre-close hooks at 1731595271410Disabling compacts and flushes for region at 1731595271410Disabling writes for close at 1731595271411 (+1 ms)Writing region close event to WAL at 1731595271421 (+10 ms)Running coprocessor post-close hooks at 1731595271452 (+31 ms)Closed at 1731595271452 2024-11-14T14:41:11,453 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,fff,1731595263719.d35b9113ecfdcde0da6d3c51d6891527. 2024-11-14T14:41:11,453 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5e71d8bcd86c479813fc8c2e3a01b917, disabling compactions & flushes 2024-11-14T14:41:11,453 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,eee,1731595263719.5e71d8bcd86c479813fc8c2e3a01b917. 2024-11-14T14:41:11,453 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1731595263719.5e71d8bcd86c479813fc8c2e3a01b917. 2024-11-14T14:41:11,453 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,eee,1731595263719.5e71d8bcd86c479813fc8c2e3a01b917. after waiting 0 ms 2024-11-14T14:41:11,453 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,eee,1731595263719.5e71d8bcd86c479813fc8c2e3a01b917. 2024-11-14T14:41:11,478 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/8dbd58303faefb2f8b5972b0c3fc3aea/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T14:41:11,479 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ddd,1731595263719.8dbd58303faefb2f8b5972b0c3fc3aea. 2024-11-14T14:41:11,479 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8dbd58303faefb2f8b5972b0c3fc3aea: Waiting for close lock at 1731595271427Running coprocessor pre-close hooks at 1731595271427Disabling compacts and flushes for region at 1731595271427Disabling writes for close at 1731595271427Writing region close event to WAL at 1731595271460 (+33 ms)Running coprocessor post-close hooks at 1731595271479 (+19 ms)Closed at 1731595271479 2024-11-14T14:41:11,479 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ddd,1731595263719.8dbd58303faefb2f8b5972b0c3fc3aea. 2024-11-14T14:41:11,480 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3613f06851ef4465281b91596c2f4aa4, disabling compactions & flushes 2024-11-14T14:41:11,480 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,,1731595263719.3613f06851ef4465281b91596c2f4aa4. 2024-11-14T14:41:11,480 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1731595263719.3613f06851ef4465281b91596c2f4aa4. 2024-11-14T14:41:11,480 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,,1731595263719.3613f06851ef4465281b91596c2f4aa4. after waiting 0 ms 2024-11-14T14:41:11,480 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,,1731595263719.3613f06851ef4465281b91596c2f4aa4. 2024-11-14T14:41:11,487 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/3613f06851ef4465281b91596c2f4aa4/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T14:41:11,487 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,,1731595263719.3613f06851ef4465281b91596c2f4aa4. 2024-11-14T14:41:11,487 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3613f06851ef4465281b91596c2f4aa4: Waiting for close lock at 1731595271480Running coprocessor pre-close hooks at 1731595271480Disabling compacts and flushes for region at 1731595271480Disabling writes for close at 1731595271480Writing region close event to WAL at 1731595271481 (+1 ms)Running coprocessor post-close hooks at 1731595271487 (+6 ms)Closed at 1731595271487 2024-11-14T14:41:11,487 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,,1731595263719.3613f06851ef4465281b91596c2f4aa4. 2024-11-14T14:41:11,488 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ad7ad94ee453836a534ae596990b42b8, disabling compactions & flushes 2024-11-14T14:41:11,488 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,hhh,1731595263719.ad7ad94ee453836a534ae596990b42b8. 2024-11-14T14:41:11,488 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1731595263719.ad7ad94ee453836a534ae596990b42b8. 2024-11-14T14:41:11,488 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1731595263719.ad7ad94ee453836a534ae596990b42b8. after waiting 0 ms 2024-11-14T14:41:11,488 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1731595263719.ad7ad94ee453836a534ae596990b42b8. 2024-11-14T14:41:11,503 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/hbase/meta/1588230740/recovered.edits/74.seqid, newMaxSeqId=74, maxSeqId=60 2024-11-14T14:41:11,503 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-14T14:41:11,504 INFO [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T14:41:11,504 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731595271229Running coprocessor pre-close hooks at 1731595271229Disabling compacts and flushes for region at 1731595271229Disabling writes for close at 1731595271230 (+1 ms)Obtaining lock to block concurrent updates at 1731595271236 (+6 ms)Preparing flush snapshotting stores in 1588230740 at 1731595271236Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=2720, getHeapSize=5504, getOffHeapSize=0, getCellsCount=24 at 1731595271248 (+12 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731595271250 (+2 ms)Flushing 1588230740/info: creating writer at 1731595271250Flushing 1588230740/info: appending metadata at 1731595271261 (+11 ms)Flushing 1588230740/info: closing flushed file at 1731595271262 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@734c6562: reopening flushed file at 1731595271353 (+91 ms)Finished flush of dataSize ~2.66 KB/2720, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 158ms, sequenceid=71, compaction requested=false at 1731595271388 (+35 ms)Writing region close event to WAL at 1731595271466 (+78 ms)Running coprocessor post-close hooks at 1731595271503 (+37 ms)Closed at 1731595271504 (+1 ms) 2024-11-14T14:41:11,504 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-14T14:41:11,524 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/5e71d8bcd86c479813fc8c2e3a01b917/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T14:41:11,525 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,eee,1731595263719.5e71d8bcd86c479813fc8c2e3a01b917. 2024-11-14T14:41:11,525 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5e71d8bcd86c479813fc8c2e3a01b917: Waiting for close lock at 1731595271453Running coprocessor pre-close hooks at 1731595271453Disabling compacts and flushes for region at 1731595271453Disabling writes for close at 1731595271453Writing region close event to WAL at 1731595271466 (+13 ms)Running coprocessor post-close hooks at 1731595271525 (+59 ms)Closed at 1731595271525 2024-11-14T14:41:11,526 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,eee,1731595263719.5e71d8bcd86c479813fc8c2e3a01b917. 2024-11-14T14:41:11,526 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9cb318690a770d1645dfd64bec1a284b, disabling compactions & flushes 2024-11-14T14:41:11,526 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ggg,1731595263719.9cb318690a770d1645dfd64bec1a284b. 2024-11-14T14:41:11,526 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1731595263719.9cb318690a770d1645dfd64bec1a284b. 2024-11-14T14:41:11,526 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1731595263719.9cb318690a770d1645dfd64bec1a284b. after waiting 0 ms 2024-11-14T14:41:11,526 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1731595263719.9cb318690a770d1645dfd64bec1a284b. 2024-11-14T14:41:11,529 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/ad7ad94ee453836a534ae596990b42b8/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T14:41:11,530 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,hhh,1731595263719.ad7ad94ee453836a534ae596990b42b8. 2024-11-14T14:41:11,530 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ad7ad94ee453836a534ae596990b42b8: Waiting for close lock at 1731595271488Running coprocessor pre-close hooks at 1731595271488Disabling compacts and flushes for region at 1731595271488Disabling writes for close at 1731595271488Writing region close event to WAL at 1731595271488Running coprocessor post-close hooks at 1731595271530 (+42 ms)Closed at 1731595271530 2024-11-14T14:41:11,531 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,hhh,1731595263719.ad7ad94ee453836a534ae596990b42b8. 2024-11-14T14:41:11,531 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6d725ecb42ad6f19c89d6ddaa229edc8, disabling compactions & flushes 2024-11-14T14:41:11,531 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,uuu,1731595263719.6d725ecb42ad6f19c89d6ddaa229edc8. 2024-11-14T14:41:11,531 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1731595263719.6d725ecb42ad6f19c89d6ddaa229edc8. 2024-11-14T14:41:11,531 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1731595263719.6d725ecb42ad6f19c89d6ddaa229edc8. after waiting 0 ms 2024-11-14T14:41:11,531 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1731595263719.6d725ecb42ad6f19c89d6ddaa229edc8. 2024-11-14T14:41:11,541 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/9cb318690a770d1645dfd64bec1a284b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T14:41:11,542 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ggg,1731595263719.9cb318690a770d1645dfd64bec1a284b. 2024-11-14T14:41:11,542 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9cb318690a770d1645dfd64bec1a284b: Waiting for close lock at 1731595271526Running coprocessor pre-close hooks at 1731595271526Disabling compacts and flushes for region at 1731595271526Disabling writes for close at 1731595271526Writing region close event to WAL at 1731595271530 (+4 ms)Running coprocessor post-close hooks at 1731595271542 (+12 ms)Closed at 1731595271542 2024-11-14T14:41:11,542 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ggg,1731595263719.9cb318690a770d1645dfd64bec1a284b. 2024-11-14T14:41:11,542 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1316438481dbf11d64fb665f6cdd4eb6, disabling compactions & flushes 2024-11-14T14:41:11,542 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,zzz,1731595263719.1316438481dbf11d64fb665f6cdd4eb6. 2024-11-14T14:41:11,542 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1731595263719.1316438481dbf11d64fb665f6cdd4eb6. 2024-11-14T14:41:11,542 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1731595263719.1316438481dbf11d64fb665f6cdd4eb6. after waiting 0 ms 2024-11-14T14:41:11,542 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1731595263719.1316438481dbf11d64fb665f6cdd4eb6. 2024-11-14T14:41:11,548 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/6d725ecb42ad6f19c89d6ddaa229edc8/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T14:41:11,548 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,uuu,1731595263719.6d725ecb42ad6f19c89d6ddaa229edc8. 2024-11-14T14:41:11,549 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6d725ecb42ad6f19c89d6ddaa229edc8: Waiting for close lock at 1731595271531Running coprocessor pre-close hooks at 1731595271531Disabling compacts and flushes for region at 1731595271531Disabling writes for close at 1731595271531Writing region close event to WAL at 1731595271540 (+9 ms)Running coprocessor post-close hooks at 1731595271548 (+8 ms)Closed at 1731595271548 2024-11-14T14:41:11,549 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,uuu,1731595263719.6d725ecb42ad6f19c89d6ddaa229edc8. 2024-11-14T14:41:11,549 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2e2ab5762f8d8f31683b4f4994048f3e, disabling compactions & flushes 2024-11-14T14:41:11,549 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,vvv,1731595263719.2e2ab5762f8d8f31683b4f4994048f3e. 2024-11-14T14:41:11,549 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1731595263719.2e2ab5762f8d8f31683b4f4994048f3e. 2024-11-14T14:41:11,549 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1731595263719.2e2ab5762f8d8f31683b4f4994048f3e. after waiting 0 ms 2024-11-14T14:41:11,549 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1731595263719.2e2ab5762f8d8f31683b4f4994048f3e. 2024-11-14T14:41:11,557 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/1316438481dbf11d64fb665f6cdd4eb6/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T14:41:11,558 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,zzz,1731595263719.1316438481dbf11d64fb665f6cdd4eb6. 2024-11-14T14:41:11,558 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1316438481dbf11d64fb665f6cdd4eb6: Waiting for close lock at 1731595271542Running coprocessor pre-close hooks at 1731595271542Disabling compacts and flushes for region at 1731595271542Disabling writes for close at 1731595271542Writing region close event to WAL at 1731595271545 (+3 ms)Running coprocessor post-close hooks at 1731595271558 (+13 ms)Closed at 1731595271558 2024-11-14T14:41:11,558 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,zzz,1731595263719.1316438481dbf11d64fb665f6cdd4eb6. 2024-11-14T14:41:11,558 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9f8e87df66c884919328ba09109d819c, disabling compactions & flushes 2024-11-14T14:41:11,558 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,mmm,1731595263719.9f8e87df66c884919328ba09109d819c. 2024-11-14T14:41:11,558 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1731595263719.9f8e87df66c884919328ba09109d819c. 2024-11-14T14:41:11,559 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1731595263719.9f8e87df66c884919328ba09109d819c. after waiting 0 ms 2024-11-14T14:41:11,559 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1731595263719.9f8e87df66c884919328ba09109d819c. 2024-11-14T14:41:11,564 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/2e2ab5762f8d8f31683b4f4994048f3e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T14:41:11,567 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,vvv,1731595263719.2e2ab5762f8d8f31683b4f4994048f3e. 2024-11-14T14:41:11,567 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2e2ab5762f8d8f31683b4f4994048f3e: Waiting for close lock at 1731595271549Running coprocessor pre-close hooks at 1731595271549Disabling compacts and flushes for region at 1731595271549Disabling writes for close at 1731595271549Writing region close event to WAL at 1731595271553 (+4 ms)Running coprocessor post-close hooks at 1731595271567 (+14 ms)Closed at 1731595271567 2024-11-14T14:41:11,567 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,vvv,1731595263719.2e2ab5762f8d8f31683b4f4994048f3e. 2024-11-14T14:41:11,567 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 550a3973a29751310a867312ba0b03d1, disabling compactions & flushes 2024-11-14T14:41:11,568 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ooo,1731595263719.550a3973a29751310a867312ba0b03d1. 2024-11-14T14:41:11,568 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1731595263719.550a3973a29751310a867312ba0b03d1. 2024-11-14T14:41:11,568 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1731595263719.550a3973a29751310a867312ba0b03d1. after waiting 0 ms 2024-11-14T14:41:11,568 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1731595263719.550a3973a29751310a867312ba0b03d1. 2024-11-14T14:41:11,597 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/9f8e87df66c884919328ba09109d819c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T14:41:11,598 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,mmm,1731595263719.9f8e87df66c884919328ba09109d819c. 2024-11-14T14:41:11,598 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9f8e87df66c884919328ba09109d819c: Waiting for close lock at 1731595271558Running coprocessor pre-close hooks at 1731595271558Disabling compacts and flushes for region at 1731595271558Disabling writes for close at 1731595271559 (+1 ms)Writing region close event to WAL at 1731595271572 (+13 ms)Running coprocessor post-close hooks at 1731595271598 (+26 ms)Closed at 1731595271598 2024-11-14T14:41:11,599 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,mmm,1731595263719.9f8e87df66c884919328ba09109d819c. 2024-11-14T14:41:11,599 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 71350ab931ae29ff2210bd9d6c3eeac5, disabling compactions & flushes 2024-11-14T14:41:11,599 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ttt,1731595263719.71350ab931ae29ff2210bd9d6c3eeac5. 2024-11-14T14:41:11,599 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1731595263719.71350ab931ae29ff2210bd9d6c3eeac5. 2024-11-14T14:41:11,599 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1731595263719.71350ab931ae29ff2210bd9d6c3eeac5. after waiting 0 ms 2024-11-14T14:41:11,599 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1731595263719.71350ab931ae29ff2210bd9d6c3eeac5. 2024-11-14T14:41:11,608 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/550a3973a29751310a867312ba0b03d1/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T14:41:11,609 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ooo,1731595263719.550a3973a29751310a867312ba0b03d1. 2024-11-14T14:41:11,609 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 550a3973a29751310a867312ba0b03d1: Waiting for close lock at 1731595271567Running coprocessor pre-close hooks at 1731595271567Disabling compacts and flushes for region at 1731595271567Disabling writes for close at 1731595271568 (+1 ms)Writing region close event to WAL at 1731595271580 (+12 ms)Running coprocessor post-close hooks at 1731595271609 (+29 ms)Closed at 1731595271609 2024-11-14T14:41:11,609 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ooo,1731595263719.550a3973a29751310a867312ba0b03d1. 2024-11-14T14:41:11,610 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 954bd925c25bfdb6e075a7471bbb1c4d, disabling compactions & flushes 2024-11-14T14:41:11,610 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,xxx,1731595263719.954bd925c25bfdb6e075a7471bbb1c4d. 2024-11-14T14:41:11,610 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1731595263719.954bd925c25bfdb6e075a7471bbb1c4d. 2024-11-14T14:41:11,610 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1731595263719.954bd925c25bfdb6e075a7471bbb1c4d. after waiting 0 ms 2024-11-14T14:41:11,610 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1731595263719.954bd925c25bfdb6e075a7471bbb1c4d. 2024-11-14T14:41:11,618 DEBUG [RS:2;da1c6afcd1f9:46229 {}] regionserver.HRegionServer(1351): Waiting on 177291d3bf2c8a0b1e9a3689929e9e50, 71350ab931ae29ff2210bd9d6c3eeac5, 77f677c4c4cfcadb020d2436881cdeaa 2024-11-14T14:41:11,628 DEBUG [RS:0;da1c6afcd1f9:40103 {}] regionserver.HRegionServer(1351): Waiting on 135a286d1aafce40676bb4d4370c9daf, 954bd925c25bfdb6e075a7471bbb1c4d 2024-11-14T14:41:11,630 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/71350ab931ae29ff2210bd9d6c3eeac5/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T14:41:11,631 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ttt,1731595263719.71350ab931ae29ff2210bd9d6c3eeac5. 2024-11-14T14:41:11,631 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 71350ab931ae29ff2210bd9d6c3eeac5: Waiting for close lock at 1731595271599Running coprocessor pre-close hooks at 1731595271599Disabling compacts and flushes for region at 1731595271599Disabling writes for close at 1731595271599Writing region close event to WAL at 1731595271613 (+14 ms)Running coprocessor post-close hooks at 1731595271631 (+18 ms)Closed at 1731595271631 2024-11-14T14:41:11,631 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ttt,1731595263719.71350ab931ae29ff2210bd9d6c3eeac5. 2024-11-14T14:41:11,631 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 77f677c4c4cfcadb020d2436881cdeaa, disabling compactions & flushes 2024-11-14T14:41:11,631 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,lll,1731595263719.77f677c4c4cfcadb020d2436881cdeaa. 2024-11-14T14:41:11,632 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1731595263719.77f677c4c4cfcadb020d2436881cdeaa. 2024-11-14T14:41:11,632 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,lll,1731595263719.77f677c4c4cfcadb020d2436881cdeaa. after waiting 0 ms 2024-11-14T14:41:11,632 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,lll,1731595263719.77f677c4c4cfcadb020d2436881cdeaa. 2024-11-14T14:41:11,637 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/954bd925c25bfdb6e075a7471bbb1c4d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T14:41:11,638 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,xxx,1731595263719.954bd925c25bfdb6e075a7471bbb1c4d. 2024-11-14T14:41:11,638 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 954bd925c25bfdb6e075a7471bbb1c4d: Waiting for close lock at 1731595271610Running coprocessor pre-close hooks at 1731595271610Disabling compacts and flushes for region at 1731595271610Disabling writes for close at 1731595271610Writing region close event to WAL at 1731595271617 (+7 ms)Running coprocessor post-close hooks at 1731595271638 (+21 ms)Closed at 1731595271638 2024-11-14T14:41:11,638 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,xxx,1731595263719.954bd925c25bfdb6e075a7471bbb1c4d. 2024-11-14T14:41:11,638 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 135a286d1aafce40676bb4d4370c9daf, disabling compactions & flushes 2024-11-14T14:41:11,639 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,iii,1731595263719.135a286d1aafce40676bb4d4370c9daf. 2024-11-14T14:41:11,639 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1731595263719.135a286d1aafce40676bb4d4370c9daf. 2024-11-14T14:41:11,639 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,iii,1731595263719.135a286d1aafce40676bb4d4370c9daf. after waiting 0 ms 2024-11-14T14:41:11,639 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,iii,1731595263719.135a286d1aafce40676bb4d4370c9daf. 2024-11-14T14:41:11,653 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/77f677c4c4cfcadb020d2436881cdeaa/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T14:41:11,654 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/135a286d1aafce40676bb4d4370c9daf/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T14:41:11,654 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,lll,1731595263719.77f677c4c4cfcadb020d2436881cdeaa. 2024-11-14T14:41:11,654 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 77f677c4c4cfcadb020d2436881cdeaa: Waiting for close lock at 1731595271631Running coprocessor pre-close hooks at 1731595271631Disabling compacts and flushes for region at 1731595271631Disabling writes for close at 1731595271632 (+1 ms)Writing region close event to WAL at 1731595271639 (+7 ms)Running coprocessor post-close hooks at 1731595271654 (+15 ms)Closed at 1731595271654 2024-11-14T14:41:11,654 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,lll,1731595263719.77f677c4c4cfcadb020d2436881cdeaa. 2024-11-14T14:41:11,654 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,iii,1731595263719.135a286d1aafce40676bb4d4370c9daf. 2024-11-14T14:41:11,655 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 135a286d1aafce40676bb4d4370c9daf: Waiting for close lock at 1731595271638Running coprocessor pre-close hooks at 1731595271638Disabling compacts and flushes for region at 1731595271638Disabling writes for close at 1731595271639 (+1 ms)Writing region close event to WAL at 1731595271648 (+9 ms)Running coprocessor post-close hooks at 1731595271654 (+6 ms)Closed at 1731595271654 2024-11-14T14:41:11,655 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 177291d3bf2c8a0b1e9a3689929e9e50, disabling compactions & flushes 2024-11-14T14:41:11,655 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,iii,1731595263719.135a286d1aafce40676bb4d4370c9daf. 2024-11-14T14:41:11,655 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,sss,1731595263719.177291d3bf2c8a0b1e9a3689929e9e50. 2024-11-14T14:41:11,655 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1731595263719.177291d3bf2c8a0b1e9a3689929e9e50. 2024-11-14T14:41:11,655 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,sss,1731595263719.177291d3bf2c8a0b1e9a3689929e9e50. after waiting 0 ms 2024-11-14T14:41:11,655 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,sss,1731595263719.177291d3bf2c8a0b1e9a3689929e9e50. 2024-11-14T14:41:11,660 INFO [regionserver/da1c6afcd1f9:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-14T14:41:11,661 INFO [regionserver/da1c6afcd1f9:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-14T14:41:11,666 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/data/default/test/177291d3bf2c8a0b1e9a3689929e9e50/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T14:41:11,667 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,sss,1731595263719.177291d3bf2c8a0b1e9a3689929e9e50. 2024-11-14T14:41:11,667 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 177291d3bf2c8a0b1e9a3689929e9e50: Waiting for close lock at 1731595271655Running coprocessor pre-close hooks at 1731595271655Disabling compacts and flushes for region at 1731595271655Disabling writes for close at 1731595271655Writing region close event to WAL at 1731595271658 (+3 ms)Running coprocessor post-close hooks at 1731595271667 (+9 ms)Closed at 1731595271667 2024-11-14T14:41:11,667 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,sss,1731595263719.177291d3bf2c8a0b1e9a3689929e9e50. 2024-11-14T14:41:11,719 INFO [regionserver/da1c6afcd1f9:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-14T14:41:11,719 INFO [regionserver/da1c6afcd1f9:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-14T14:41:11,819 INFO [RS:2;da1c6afcd1f9:46229 {}] regionserver.HRegionServer(976): stopping server da1c6afcd1f9,46229,1731595261363; all regions closed. 2024-11-14T14:41:11,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45825 is added to blk_1073741835_1011 (size=7074) 2024-11-14T14:41:11,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45021 is added to blk_1073741835_1011 (size=7074) 2024-11-14T14:41:11,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36849 is added to blk_1073741835_1011 (size=7074) 2024-11-14T14:41:11,829 INFO [RS:0;da1c6afcd1f9:40103 {}] regionserver.HRegionServer(976): stopping server da1c6afcd1f9,40103,1731595261206; all regions closed. 2024-11-14T14:41:11,833 DEBUG [RS:2;da1c6afcd1f9:46229 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/oldWALs 2024-11-14T14:41:11,834 INFO [RS:2;da1c6afcd1f9:46229 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL da1c6afcd1f9%2C46229%2C1731595261363:(num 1731595262836) 2024-11-14T14:41:11,834 DEBUG [RS:2;da1c6afcd1f9:46229 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:41:11,834 INFO [RS:2;da1c6afcd1f9:46229 {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T14:41:11,834 INFO [RS:2;da1c6afcd1f9:46229 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T14:41:11,835 INFO [RS:2;da1c6afcd1f9:46229 {}] hbase.ChoreService(370): Chore service for: regionserver/da1c6afcd1f9:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-14T14:41:11,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36849 is added to blk_1073741865_1041 (size=4204) 2024-11-14T14:41:11,836 INFO [RS:2;da1c6afcd1f9:46229 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-14T14:41:11,836 INFO [RS:2;da1c6afcd1f9:46229 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-14T14:41:11,836 INFO [RS:2;da1c6afcd1f9:46229 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-14T14:41:11,836 INFO [RS:2;da1c6afcd1f9:46229 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T14:41:11,836 INFO [RS:2;da1c6afcd1f9:46229 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46229 2024-11-14T14:41:11,837 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T14:41:11,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45825 is added to blk_1073741865_1041 (size=4204) 2024-11-14T14:41:11,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45021 is added to blk_1073741865_1041 (size=4204) 2024-11-14T14:41:11,839 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34057-0x101a73a44750000, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T14:41:11,839 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46229-0x101a73a44750003, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/da1c6afcd1f9,46229,1731595261363 2024-11-14T14:41:11,840 INFO [RS:2;da1c6afcd1f9:46229 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T14:41:11,841 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [da1c6afcd1f9,46229,1731595261363] 2024-11-14T14:41:11,842 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/da1c6afcd1f9,46229,1731595261363 already deleted, retry=false 2024-11-14T14:41:11,843 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; da1c6afcd1f9,46229,1731595261363 expired; onlineServers=1 2024-11-14T14:41:11,846 DEBUG [RS:0;da1c6afcd1f9:40103 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/oldWALs 2024-11-14T14:41:11,846 INFO [RS:0;da1c6afcd1f9:40103 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL da1c6afcd1f9%2C40103%2C1731595261206.meta:.meta(num 1731595267258) 2024-11-14T14:41:11,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45021 is added to blk_1073741833_1009 (size=7064) 2024-11-14T14:41:11,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36849 is added to blk_1073741833_1009 (size=7064) 2024-11-14T14:41:11,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45825 is added to blk_1073741833_1009 (size=7064) 2024-11-14T14:41:11,858 DEBUG [RS:0;da1c6afcd1f9:40103 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/oldWALs 2024-11-14T14:41:11,858 INFO [RS:0;da1c6afcd1f9:40103 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL da1c6afcd1f9%2C40103%2C1731595261206:(num 1731595262835) 2024-11-14T14:41:11,858 DEBUG [RS:0;da1c6afcd1f9:40103 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:41:11,858 INFO [RS:0;da1c6afcd1f9:40103 {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T14:41:11,858 INFO [RS:0;da1c6afcd1f9:40103 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T14:41:11,858 INFO [RS:0;da1c6afcd1f9:40103 {}] hbase.ChoreService(370): Chore service for: regionserver/da1c6afcd1f9:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-14T14:41:11,859 INFO [RS:0;da1c6afcd1f9:40103 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T14:41:11,859 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T14:41:11,859 INFO [RS:0;da1c6afcd1f9:40103 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40103 2024-11-14T14:41:11,861 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40103-0x101a73a44750001, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/da1c6afcd1f9,40103,1731595261206 2024-11-14T14:41:11,861 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34057-0x101a73a44750000, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T14:41:11,861 INFO [RS:0;da1c6afcd1f9:40103 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T14:41:11,862 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [da1c6afcd1f9,40103,1731595261206] 2024-11-14T14:41:11,863 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/da1c6afcd1f9,40103,1731595261206 already deleted, retry=false 2024-11-14T14:41:11,864 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; da1c6afcd1f9,40103,1731595261206 expired; onlineServers=0 2024-11-14T14:41:11,864 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'da1c6afcd1f9,34057,1731595260501' ***** 2024-11-14T14:41:11,864 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-14T14:41:11,864 INFO [M:0;da1c6afcd1f9:34057 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T14:41:11,864 INFO [M:0;da1c6afcd1f9:34057 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T14:41:11,864 DEBUG [M:0;da1c6afcd1f9:34057 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-14T14:41:11,864 DEBUG [M:0;da1c6afcd1f9:34057 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-14T14:41:11,864 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-14T14:41:11,864 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster-HFileCleaner.large.0-1731595262486 {}] cleaner.HFileCleaner(306): Exit Thread[master/da1c6afcd1f9:0:becomeActiveMaster-HFileCleaner.large.0-1731595262486,5,FailOnTimeoutGroup] 2024-11-14T14:41:11,865 INFO [M:0;da1c6afcd1f9:34057 {}] hbase.ChoreService(370): Chore service for: master/da1c6afcd1f9:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-14T14:41:11,864 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster-HFileCleaner.small.0-1731595262490 {}] cleaner.HFileCleaner(306): Exit Thread[master/da1c6afcd1f9:0:becomeActiveMaster-HFileCleaner.small.0-1731595262490,5,FailOnTimeoutGroup] 2024-11-14T14:41:11,865 INFO [M:0;da1c6afcd1f9:34057 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T14:41:11,865 DEBUG [M:0;da1c6afcd1f9:34057 {}] master.HMaster(1795): Stopping service threads 2024-11-14T14:41:11,865 INFO [M:0;da1c6afcd1f9:34057 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-14T14:41:11,865 INFO [M:0;da1c6afcd1f9:34057 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T14:41:11,866 INFO [M:0;da1c6afcd1f9:34057 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T14:41:11,866 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-14T14:41:11,866 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34057-0x101a73a44750000, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-14T14:41:11,866 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34057-0x101a73a44750000, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:41:11,867 DEBUG [M:0;da1c6afcd1f9:34057 {}] zookeeper.ZKUtil(347): master:34057-0x101a73a44750000, quorum=127.0.0.1:60650, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-14T14:41:11,867 WARN [M:0;da1c6afcd1f9:34057 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-14T14:41:11,868 INFO [M:0;da1c6afcd1f9:34057 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/.lastflushedseqids 2024-11-14T14:41:11,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45021 is added to blk_1073741870_1046 (size=1) 2024-11-14T14:41:11,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45825 is added to blk_1073741870_1046 (size=1) 2024-11-14T14:41:11,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36849 is added to blk_1073741870_1046 (size=1) 2024-11-14T14:41:11,911 INFO [M:0;da1c6afcd1f9:34057 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T14:41:11,911 INFO [M:0;da1c6afcd1f9:34057 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-14T14:41:11,912 DEBUG [M:0;da1c6afcd1f9:34057 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-14T14:41:11,912 INFO [M:0;da1c6afcd1f9:34057 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:41:11,912 DEBUG [M:0;da1c6afcd1f9:34057 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:41:11,912 DEBUG [M:0;da1c6afcd1f9:34057 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-14T14:41:11,912 DEBUG [M:0;da1c6afcd1f9:34057 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:41:11,912 INFO [M:0;da1c6afcd1f9:34057 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=249.77 KB heapSize=303.53 KB 2024-11-14T14:41:11,938 DEBUG [M:0;da1c6afcd1f9:34057 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/503e8e7923254ab89e4a46f058acd969 is 82, key is hbase:meta,,1/info:regioninfo/1731595267957/Put/seqid=0 2024-11-14T14:41:11,942 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46229-0x101a73a44750003, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T14:41:11,942 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46229-0x101a73a44750003, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T14:41:11,942 INFO [RS:2;da1c6afcd1f9:46229 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T14:41:11,942 INFO [RS:2;da1c6afcd1f9:46229 {}] regionserver.HRegionServer(1031): Exiting; stopping=da1c6afcd1f9,46229,1731595261363; zookeeper connection closed. 2024-11-14T14:41:11,947 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5c115ab6 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5c115ab6 2024-11-14T14:41:11,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36849 is added to blk_1073741871_1047 (size=6071) 2024-11-14T14:41:11,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45021 is added to blk_1073741871_1047 (size=6071) 2024-11-14T14:41:11,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45825 is added to blk_1073741871_1047 (size=6071) 2024-11-14T14:41:11,957 INFO [M:0;da1c6afcd1f9:34057 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=574 (bloomFilter=true), to=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/503e8e7923254ab89e4a46f058acd969 2024-11-14T14:41:11,964 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40103-0x101a73a44750001, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T14:41:11,964 INFO [RS:0;da1c6afcd1f9:40103 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T14:41:11,964 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40103-0x101a73a44750001, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T14:41:11,964 INFO [RS:0;da1c6afcd1f9:40103 {}] regionserver.HRegionServer(1031): Exiting; stopping=da1c6afcd1f9,40103,1731595261206; zookeeper connection closed. 2024-11-14T14:41:11,964 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@29ed8d34 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@29ed8d34 2024-11-14T14:41:11,965 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-14T14:41:11,992 DEBUG [M:0;da1c6afcd1f9:34057 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/83a58c4871d74394ac18f68d65cd8f2d is 2990, key is \x00\x00\x00\x00\x00\x00\x009/proc:d/1731595271130/Put/seqid=0 2024-11-14T14:41:12,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45825 is added to blk_1073741872_1048 (size=12801) 2024-11-14T14:41:12,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45021 is added to blk_1073741872_1048 (size=12801) 2024-11-14T14:41:12,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36849 is added to blk_1073741872_1048 (size=12801) 2024-11-14T14:41:12,003 INFO [M:0;da1c6afcd1f9:34057 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=248.50 KB at sequenceid=574 (bloomFilter=true), to=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/83a58c4871d74394ac18f68d65cd8f2d 2024-11-14T14:41:12,042 DEBUG [M:0;da1c6afcd1f9:34057 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f2f6bc99c6754a17bc398fe35c589e13 is 69, key is da1c6afcd1f9,40103,1731595261206/rs:state/1731595262544/Put/seqid=0 2024-11-14T14:41:12,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45021 is added to blk_1073741873_1049 (size=5442) 2024-11-14T14:41:12,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36849 is added to blk_1073741873_1049 (size=5442) 2024-11-14T14:41:12,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45825 is added to blk_1073741873_1049 (size=5442) 2024-11-14T14:41:12,062 INFO [M:0;da1c6afcd1f9:34057 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=574 (bloomFilter=true), to=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f2f6bc99c6754a17bc398fe35c589e13 2024-11-14T14:41:12,070 INFO [M:0;da1c6afcd1f9:34057 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f2f6bc99c6754a17bc398fe35c589e13 2024-11-14T14:41:12,092 DEBUG [M:0;da1c6afcd1f9:34057 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/20c28f1c04fb4c0da91d27e40b4df83f is 52, key is load_balancer_on/state:d/1731595266260/Put/seqid=0 2024-11-14T14:41:12,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45825 is added to blk_1073741874_1050 (size=5058) 2024-11-14T14:41:12,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36849 is added to blk_1073741874_1050 (size=5058) 2024-11-14T14:41:12,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45021 is added to blk_1073741874_1050 (size=5058) 2024-11-14T14:41:12,115 INFO [M:0;da1c6afcd1f9:34057 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=574 (bloomFilter=true), to=hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/20c28f1c04fb4c0da91d27e40b4df83f 2024-11-14T14:41:12,129 DEBUG [M:0;da1c6afcd1f9:34057 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/503e8e7923254ab89e4a46f058acd969 as hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/503e8e7923254ab89e4a46f058acd969 2024-11-14T14:41:12,142 INFO [M:0;da1c6afcd1f9:34057 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/503e8e7923254ab89e4a46f058acd969, entries=14, sequenceid=574, filesize=5.9 K 2024-11-14T14:41:12,145 DEBUG [M:0;da1c6afcd1f9:34057 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/83a58c4871d74394ac18f68d65cd8f2d as hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/83a58c4871d74394ac18f68d65cd8f2d 2024-11-14T14:41:12,157 INFO [M:0;da1c6afcd1f9:34057 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/83a58c4871d74394ac18f68d65cd8f2d, entries=79, sequenceid=574, filesize=12.5 K 2024-11-14T14:41:12,160 DEBUG [M:0;da1c6afcd1f9:34057 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f2f6bc99c6754a17bc398fe35c589e13 as hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f2f6bc99c6754a17bc398fe35c589e13 2024-11-14T14:41:12,176 INFO [M:0;da1c6afcd1f9:34057 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f2f6bc99c6754a17bc398fe35c589e13 2024-11-14T14:41:12,177 INFO [M:0;da1c6afcd1f9:34057 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f2f6bc99c6754a17bc398fe35c589e13, entries=3, sequenceid=574, filesize=5.3 K 2024-11-14T14:41:12,178 DEBUG [M:0;da1c6afcd1f9:34057 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/20c28f1c04fb4c0da91d27e40b4df83f as hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/20c28f1c04fb4c0da91d27e40b4df83f 2024-11-14T14:41:12,186 INFO [M:0;da1c6afcd1f9:34057 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37365/user/jenkins/test-data/c7ebeaed-7124-bdf1-f592-f3da03b09d5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/20c28f1c04fb4c0da91d27e40b4df83f, entries=1, sequenceid=574, filesize=4.9 K 2024-11-14T14:41:12,187 INFO [M:0;da1c6afcd1f9:34057 {}] regionserver.HRegion(3140): Finished flush of dataSize ~249.77 KB/255768, heapSize ~303.47 KB/310752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 275ms, sequenceid=574, compaction requested=false 2024-11-14T14:41:12,195 INFO [M:0;da1c6afcd1f9:34057 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:41:12,196 DEBUG [M:0;da1c6afcd1f9:34057 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731595271912Disabling compacts and flushes for region at 1731595271912Disabling writes for close at 1731595271912Obtaining lock to block concurrent updates at 1731595271912Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731595271912Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=255768, getHeapSize=310752, getOffHeapSize=0, getCellsCount=713 at 1731595271913 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731595271914 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731595271914Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731595271938 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731595271938Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731595271965 (+27 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731595271991 (+26 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731595271991Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731595272015 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731595272040 (+25 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731595272040Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731595272070 (+30 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731595272092 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731595272092Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6dfdbb5c: reopening flushed file at 1731595272127 (+35 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@48d224aa: reopening flushed file at 1731595272142 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@579e1613: reopening flushed file at 1731595272158 (+16 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a185a09: reopening flushed file at 1731595272177 (+19 ms)Finished flush of dataSize ~249.77 KB/255768, heapSize ~303.47 KB/310752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 275ms, sequenceid=574, compaction requested=false at 1731595272187 (+10 ms)Writing region close event to WAL at 1731595272195 (+8 ms)Closed at 1731595272195 2024-11-14T14:41:12,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45021 is added to blk_1073741830_1006 (size=294087) 2024-11-14T14:41:12,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36849 is added to blk_1073741830_1006 (size=294087) 2024-11-14T14:41:12,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45825 is added to blk_1073741830_1006 (size=294087) 2024-11-14T14:41:12,201 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T14:41:12,201 INFO [M:0;da1c6afcd1f9:34057 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-14T14:41:12,202 INFO [M:0;da1c6afcd1f9:34057 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34057 2024-11-14T14:41:12,202 INFO [M:0;da1c6afcd1f9:34057 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T14:41:12,304 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34057-0x101a73a44750000, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T14:41:12,304 INFO [M:0;da1c6afcd1f9:34057 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T14:41:12,304 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34057-0x101a73a44750000, quorum=127.0.0.1:60650, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T14:41:12,314 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@46041a9d{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T14:41:12,317 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2a39ea25{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T14:41:12,317 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T14:41:12,317 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@48a2d1c6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T14:41:12,317 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6ed43cd1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/694aaacd-0466-2f0d-621d-715bf1fdc22e/hadoop.log.dir/,STOPPED} 2024-11-14T14:41:12,321 WARN [BP-986035353-172.17.0.2-1731595256759 heartbeating to localhost/127.0.0.1:37365 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-14T14:41:12,320 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-14T14:41:12,321 WARN [BP-986035353-172.17.0.2-1731595256759 heartbeating to localhost/127.0.0.1:37365 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-986035353-172.17.0.2-1731595256759 (Datanode Uuid d7d7a462-f4b1-406b-b08e-a0cbd03c121e) service to localhost/127.0.0.1:37365 2024-11-14T14:41:12,321 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-14T14:41:12,322 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/694aaacd-0466-2f0d-621d-715bf1fdc22e/cluster_9a8ac74c-702c-79fa-009c-bd4fec459006/data/data5/current/BP-986035353-172.17.0.2-1731595256759 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T14:41:12,322 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/694aaacd-0466-2f0d-621d-715bf1fdc22e/cluster_9a8ac74c-702c-79fa-009c-bd4fec459006/data/data6/current/BP-986035353-172.17.0.2-1731595256759 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T14:41:12,323 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-14T14:41:12,326 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@395ff33a{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T14:41:12,326 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4d24ac65{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T14:41:12,326 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T14:41:12,326 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6331ed3f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T14:41:12,326 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2ff36c4c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/694aaacd-0466-2f0d-621d-715bf1fdc22e/hadoop.log.dir/,STOPPED} 2024-11-14T14:41:12,328 WARN [BP-986035353-172.17.0.2-1731595256759 heartbeating to localhost/127.0.0.1:37365 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-14T14:41:12,328 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-14T14:41:12,328 WARN [BP-986035353-172.17.0.2-1731595256759 heartbeating to localhost/127.0.0.1:37365 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-986035353-172.17.0.2-1731595256759 (Datanode Uuid 0c2cd0b9-47f4-4235-8c22-bbadfd149574) service to localhost/127.0.0.1:37365 2024-11-14T14:41:12,328 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-14T14:41:12,329 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/694aaacd-0466-2f0d-621d-715bf1fdc22e/cluster_9a8ac74c-702c-79fa-009c-bd4fec459006/data/data3/current/BP-986035353-172.17.0.2-1731595256759 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T14:41:12,329 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/694aaacd-0466-2f0d-621d-715bf1fdc22e/cluster_9a8ac74c-702c-79fa-009c-bd4fec459006/data/data4/current/BP-986035353-172.17.0.2-1731595256759 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T14:41:12,330 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-14T14:41:12,334 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@65e17246{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T14:41:12,334 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@220c8d80{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T14:41:12,334 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T14:41:12,334 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@c7135d9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T14:41:12,335 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@51c39823{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/694aaacd-0466-2f0d-621d-715bf1fdc22e/hadoop.log.dir/,STOPPED} 2024-11-14T14:41:12,336 WARN [BP-986035353-172.17.0.2-1731595256759 heartbeating to localhost/127.0.0.1:37365 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-14T14:41:12,336 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-14T14:41:12,336 WARN [BP-986035353-172.17.0.2-1731595256759 heartbeating to localhost/127.0.0.1:37365 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-986035353-172.17.0.2-1731595256759 (Datanode Uuid 967fa7ac-b139-4ef6-9710-d38aa72e25ab) service to localhost/127.0.0.1:37365 2024-11-14T14:41:12,336 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-14T14:41:12,337 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/694aaacd-0466-2f0d-621d-715bf1fdc22e/cluster_9a8ac74c-702c-79fa-009c-bd4fec459006/data/data1/current/BP-986035353-172.17.0.2-1731595256759 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T14:41:12,337 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/694aaacd-0466-2f0d-621d-715bf1fdc22e/cluster_9a8ac74c-702c-79fa-009c-bd4fec459006/data/data2/current/BP-986035353-172.17.0.2-1731595256759 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T14:41:12,337 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-14T14:41:12,346 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@64418798{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-14T14:41:12,347 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@11744524{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T14:41:12,347 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T14:41:12,347 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@595320c8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T14:41:12,347 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3912242{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/694aaacd-0466-2f0d-621d-715bf1fdc22e/hadoop.log.dir/,STOPPED} 2024-11-14T14:41:12,362 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-14T14:41:12,425 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down