2024-11-25 03:41:05,696 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-25 03:41:05,711 main DEBUG Took 0.012440 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-25 03:41:05,711 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-25 03:41:05,712 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-25 03:41:05,712 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-25 03:41:05,713 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:41:05,722 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-25 03:41:05,738 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:41:05,740 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:41:05,741 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:41:05,742 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:41:05,742 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:41:05,743 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:41:05,744 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:41:05,745 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:41:05,745 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:41:05,746 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:41:05,747 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:41:05,748 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:41:05,748 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:41:05,749 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:41:05,750 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:41:05,750 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:41:05,751 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:41:05,751 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:41:05,752 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:41:05,752 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:41:05,753 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:41:05,753 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:41:05,754 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:41:05,754 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:41:05,755 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:41:05,755 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-25 03:41:05,757 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:41:05,759 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-25 03:41:05,761 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-25 03:41:05,762 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-25 03:41:05,764 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-25 03:41:05,764 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-25 03:41:05,775 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-25 03:41:05,778 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-25 03:41:05,780 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-25 03:41:05,781 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-25 03:41:05,781 main DEBUG createAppenders(={Console}) 2024-11-25 03:41:05,782 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 initialized 2024-11-25 03:41:05,783 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-25 03:41:05,783 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 OK. 2024-11-25 03:41:05,784 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-25 03:41:05,784 main DEBUG OutputStream closed 2024-11-25 03:41:05,785 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-25 03:41:05,785 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-25 03:41:05,785 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@c88a337 OK 2024-11-25 03:41:05,857 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-25 03:41:05,859 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-25 03:41:05,860 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-25 03:41:05,860 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-25 03:41:05,861 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-25 03:41:05,861 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-25 03:41:05,862 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-25 03:41:05,862 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-25 03:41:05,862 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-25 03:41:05,863 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-25 03:41:05,863 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-25 03:41:05,863 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-25 03:41:05,864 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-25 03:41:05,864 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-25 03:41:05,864 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-25 03:41:05,865 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-25 03:41:05,865 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-25 03:41:05,866 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-25 03:41:05,868 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-25 03:41:05,868 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@13330ac6) with optional ClassLoader: null 2024-11-25 03:41:05,868 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-25 03:41:05,869 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@13330ac6] started OK. 2024-11-25T03:41:05,882 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP timeout: 13 mins 2024-11-25 03:41:05,884 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-25 03:41:05,885 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-25T03:41:06,118 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/408718d8-141d-acc8-2a26-73c097eff0c8 2024-11-25T03:41:06,137 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$HMasterForTest, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-25T03:41:06,153 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/408718d8-141d-acc8-2a26-73c097eff0c8/cluster_86ef855a-7356-d2a4-6f37-d8b759212c31, deleteOnExit=true 2024-11-25T03:41:06,153 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-25T03:41:06,154 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/408718d8-141d-acc8-2a26-73c097eff0c8/test.cache.data in system properties and HBase conf 2024-11-25T03:41:06,155 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/408718d8-141d-acc8-2a26-73c097eff0c8/hadoop.tmp.dir in system properties and HBase conf 2024-11-25T03:41:06,155 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/408718d8-141d-acc8-2a26-73c097eff0c8/hadoop.log.dir in system properties and HBase conf 2024-11-25T03:41:06,156 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/408718d8-141d-acc8-2a26-73c097eff0c8/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-25T03:41:06,157 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/408718d8-141d-acc8-2a26-73c097eff0c8/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-25T03:41:06,157 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-25T03:41:06,256 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-25T03:41:06,373 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T03:41:06,378 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/408718d8-141d-acc8-2a26-73c097eff0c8/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-25T03:41:06,379 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/408718d8-141d-acc8-2a26-73c097eff0c8/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-25T03:41:06,380 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/408718d8-141d-acc8-2a26-73c097eff0c8/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-25T03:41:06,380 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/408718d8-141d-acc8-2a26-73c097eff0c8/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-25T03:41:06,381 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/408718d8-141d-acc8-2a26-73c097eff0c8/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-25T03:41:06,382 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/408718d8-141d-acc8-2a26-73c097eff0c8/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-25T03:41:06,382 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/408718d8-141d-acc8-2a26-73c097eff0c8/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-25T03:41:06,383 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/408718d8-141d-acc8-2a26-73c097eff0c8/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-25T03:41:06,384 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/408718d8-141d-acc8-2a26-73c097eff0c8/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-25T03:41:06,385 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/408718d8-141d-acc8-2a26-73c097eff0c8/nfs.dump.dir in system properties and HBase conf 2024-11-25T03:41:06,385 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/408718d8-141d-acc8-2a26-73c097eff0c8/java.io.tmpdir in system properties and HBase conf 2024-11-25T03:41:06,386 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/408718d8-141d-acc8-2a26-73c097eff0c8/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-25T03:41:06,386 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/408718d8-141d-acc8-2a26-73c097eff0c8/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-25T03:41:06,387 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/408718d8-141d-acc8-2a26-73c097eff0c8/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-25T03:41:07,476 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-25T03:41:07,543 INFO [Time-limited test {}] log.Log(170): Logging initialized @2500ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-25T03:41:07,608 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-25T03:41:07,662 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-25T03:41:07,681 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-25T03:41:07,682 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-25T03:41:07,683 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-25T03:41:07,694 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-25T03:41:07,696 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@265b24f1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/408718d8-141d-acc8-2a26-73c097eff0c8/hadoop.log.dir/,AVAILABLE} 2024-11-25T03:41:07,697 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@76beb053{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-25T03:41:07,863 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@376fa8f3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/408718d8-141d-acc8-2a26-73c097eff0c8/java.io.tmpdir/jetty-localhost-41687-hadoop-hdfs-3_4_1-tests_jar-_-any-387125158131289657/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-25T03:41:07,869 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@672b88ea{HTTP/1.1, (http/1.1)}{localhost:41687} 2024-11-25T03:41:07,869 INFO [Time-limited test {}] server.Server(415): Started @2826ms 2024-11-25T03:41:08,397 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-25T03:41:08,404 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-25T03:41:08,405 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-25T03:41:08,405 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-25T03:41:08,406 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-25T03:41:08,407 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@739a104e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/408718d8-141d-acc8-2a26-73c097eff0c8/hadoop.log.dir/,AVAILABLE} 2024-11-25T03:41:08,407 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1c5d2af1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-25T03:41:08,508 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2226d6a6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/408718d8-141d-acc8-2a26-73c097eff0c8/java.io.tmpdir/jetty-localhost-43065-hadoop-hdfs-3_4_1-tests_jar-_-any-11745149352245412798/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-25T03:41:08,508 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1da1f773{HTTP/1.1, (http/1.1)}{localhost:43065} 2024-11-25T03:41:08,509 INFO [Time-limited test {}] server.Server(415): Started @3466ms 2024-11-25T03:41:08,556 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-25T03:41:08,660 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-25T03:41:08,664 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-25T03:41:08,668 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-25T03:41:08,669 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-25T03:41:08,669 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-25T03:41:08,670 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@34a3e1ff{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/408718d8-141d-acc8-2a26-73c097eff0c8/hadoop.log.dir/,AVAILABLE} 2024-11-25T03:41:08,671 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@12f16529{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-25T03:41:08,769 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7cfccf27{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/408718d8-141d-acc8-2a26-73c097eff0c8/java.io.tmpdir/jetty-localhost-44721-hadoop-hdfs-3_4_1-tests_jar-_-any-14433235896769798321/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-25T03:41:08,770 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6c41bbb7{HTTP/1.1, (http/1.1)}{localhost:44721} 2024-11-25T03:41:08,770 INFO [Time-limited test {}] server.Server(415): Started @3727ms 2024-11-25T03:41:08,773 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-25T03:41:08,814 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-25T03:41:08,819 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-25T03:41:08,825 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-25T03:41:08,825 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-25T03:41:08,825 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-25T03:41:08,826 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@11ebaa12{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/408718d8-141d-acc8-2a26-73c097eff0c8/hadoop.log.dir/,AVAILABLE} 2024-11-25T03:41:08,826 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@52cf89e6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-25T03:41:08,935 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@67ddfd25{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/408718d8-141d-acc8-2a26-73c097eff0c8/java.io.tmpdir/jetty-localhost-35481-hadoop-hdfs-3_4_1-tests_jar-_-any-13135931291881722997/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-25T03:41:08,936 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@30e38263{HTTP/1.1, (http/1.1)}{localhost:35481} 2024-11-25T03:41:08,936 INFO [Time-limited test {}] server.Server(415): Started @3893ms 2024-11-25T03:41:08,938 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-25T03:41:10,507 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/408718d8-141d-acc8-2a26-73c097eff0c8/cluster_86ef855a-7356-d2a4-6f37-d8b759212c31/data/data3/current/BP-1923551470-172.17.0.2-1732506066906/current, will proceed with Du for space computation calculation, 2024-11-25T03:41:10,507 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/408718d8-141d-acc8-2a26-73c097eff0c8/cluster_86ef855a-7356-d2a4-6f37-d8b759212c31/data/data2/current/BP-1923551470-172.17.0.2-1732506066906/current, will proceed with Du for space computation calculation, 2024-11-25T03:41:10,507 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/408718d8-141d-acc8-2a26-73c097eff0c8/cluster_86ef855a-7356-d2a4-6f37-d8b759212c31/data/data4/current/BP-1923551470-172.17.0.2-1732506066906/current, will proceed with Du for space computation calculation, 2024-11-25T03:41:10,507 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/408718d8-141d-acc8-2a26-73c097eff0c8/cluster_86ef855a-7356-d2a4-6f37-d8b759212c31/data/data1/current/BP-1923551470-172.17.0.2-1732506066906/current, will proceed with Du for space computation calculation, 2024-11-25T03:41:10,540 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-25T03:41:10,540 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-25T03:41:10,552 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/408718d8-141d-acc8-2a26-73c097eff0c8/cluster_86ef855a-7356-d2a4-6f37-d8b759212c31/data/data6/current/BP-1923551470-172.17.0.2-1732506066906/current, will proceed with Du for space computation calculation, 2024-11-25T03:41:10,552 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/408718d8-141d-acc8-2a26-73c097eff0c8/cluster_86ef855a-7356-d2a4-6f37-d8b759212c31/data/data5/current/BP-1923551470-172.17.0.2-1732506066906/current, will proceed with Du for space computation calculation, 2024-11-25T03:41:10,581 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-25T03:41:10,588 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x69787065f3ca4b7b with lease ID 0xd56d903a16ee84b8: Processing first storage report for DS-88ab84a9-630b-478a-9aeb-f2157b7da108 from datanode DatanodeRegistration(127.0.0.1:33637, datanodeUuid=f9487afc-4424-41cf-8aaf-dd5cb3e8689a, infoPort=35677, infoSecurePort=0, ipcPort=34203, storageInfo=lv=-57;cid=testClusterID;nsid=5586231;c=1732506066906) 2024-11-25T03:41:10,589 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x69787065f3ca4b7b with lease ID 0xd56d903a16ee84b8: from storage DS-88ab84a9-630b-478a-9aeb-f2157b7da108 node DatanodeRegistration(127.0.0.1:33637, datanodeUuid=f9487afc-4424-41cf-8aaf-dd5cb3e8689a, infoPort=35677, infoSecurePort=0, ipcPort=34203, storageInfo=lv=-57;cid=testClusterID;nsid=5586231;c=1732506066906), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-25T03:41:10,589 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x63bc2a5e778429bb with lease ID 0xd56d903a16ee84b7: Processing first storage report for DS-43ff8d69-2bf0-4a76-b42b-90f4bc41dccf from datanode DatanodeRegistration(127.0.0.1:41447, datanodeUuid=96072fc5-5785-4c8e-b352-58105511302d, infoPort=44951, infoSecurePort=0, ipcPort=42933, storageInfo=lv=-57;cid=testClusterID;nsid=5586231;c=1732506066906) 2024-11-25T03:41:10,590 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x63bc2a5e778429bb with lease ID 0xd56d903a16ee84b7: from storage DS-43ff8d69-2bf0-4a76-b42b-90f4bc41dccf node DatanodeRegistration(127.0.0.1:41447, datanodeUuid=96072fc5-5785-4c8e-b352-58105511302d, infoPort=44951, infoSecurePort=0, ipcPort=42933, storageInfo=lv=-57;cid=testClusterID;nsid=5586231;c=1732506066906), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-25T03:41:10,590 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbaa2555d8f4badfc with lease ID 0xd56d903a16ee84b6: Processing first storage report for DS-5d64405e-cf5f-4842-9c54-201a1e1e22d8 from datanode DatanodeRegistration(127.0.0.1:34311, datanodeUuid=5bfc8e74-2dd6-4804-a4c4-92b73caa2b06, infoPort=41877, infoSecurePort=0, ipcPort=38217, storageInfo=lv=-57;cid=testClusterID;nsid=5586231;c=1732506066906) 2024-11-25T03:41:10,590 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbaa2555d8f4badfc with lease ID 0xd56d903a16ee84b6: from storage DS-5d64405e-cf5f-4842-9c54-201a1e1e22d8 node DatanodeRegistration(127.0.0.1:34311, datanodeUuid=5bfc8e74-2dd6-4804-a4c4-92b73caa2b06, infoPort=41877, infoSecurePort=0, ipcPort=38217, storageInfo=lv=-57;cid=testClusterID;nsid=5586231;c=1732506066906), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-25T03:41:10,590 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x69787065f3ca4b7b with lease ID 0xd56d903a16ee84b8: Processing first storage report for DS-0709277e-378c-4ecf-9472-21eefbf669da from datanode DatanodeRegistration(127.0.0.1:33637, datanodeUuid=f9487afc-4424-41cf-8aaf-dd5cb3e8689a, infoPort=35677, infoSecurePort=0, ipcPort=34203, storageInfo=lv=-57;cid=testClusterID;nsid=5586231;c=1732506066906) 2024-11-25T03:41:10,590 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x69787065f3ca4b7b with lease ID 0xd56d903a16ee84b8: from storage DS-0709277e-378c-4ecf-9472-21eefbf669da node DatanodeRegistration(127.0.0.1:33637, datanodeUuid=f9487afc-4424-41cf-8aaf-dd5cb3e8689a, infoPort=35677, infoSecurePort=0, ipcPort=34203, storageInfo=lv=-57;cid=testClusterID;nsid=5586231;c=1732506066906), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-25T03:41:10,591 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x63bc2a5e778429bb with lease ID 0xd56d903a16ee84b7: Processing first storage report for DS-a764563d-5416-47b2-81e2-db07ee739133 from datanode DatanodeRegistration(127.0.0.1:41447, datanodeUuid=96072fc5-5785-4c8e-b352-58105511302d, infoPort=44951, infoSecurePort=0, ipcPort=42933, storageInfo=lv=-57;cid=testClusterID;nsid=5586231;c=1732506066906) 2024-11-25T03:41:10,591 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x63bc2a5e778429bb with lease ID 0xd56d903a16ee84b7: from storage DS-a764563d-5416-47b2-81e2-db07ee739133 node DatanodeRegistration(127.0.0.1:41447, datanodeUuid=96072fc5-5785-4c8e-b352-58105511302d, infoPort=44951, infoSecurePort=0, ipcPort=42933, storageInfo=lv=-57;cid=testClusterID;nsid=5586231;c=1732506066906), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-25T03:41:10,591 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbaa2555d8f4badfc with lease ID 0xd56d903a16ee84b6: Processing first storage report for DS-1268459e-b73a-45c8-aa12-4f947f700e35 from datanode DatanodeRegistration(127.0.0.1:34311, datanodeUuid=5bfc8e74-2dd6-4804-a4c4-92b73caa2b06, infoPort=41877, infoSecurePort=0, ipcPort=38217, storageInfo=lv=-57;cid=testClusterID;nsid=5586231;c=1732506066906) 2024-11-25T03:41:10,591 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbaa2555d8f4badfc with lease ID 0xd56d903a16ee84b6: from storage DS-1268459e-b73a-45c8-aa12-4f947f700e35 node DatanodeRegistration(127.0.0.1:34311, datanodeUuid=5bfc8e74-2dd6-4804-a4c4-92b73caa2b06, infoPort=41877, infoSecurePort=0, ipcPort=38217, storageInfo=lv=-57;cid=testClusterID;nsid=5586231;c=1732506066906), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-25T03:41:10,657 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/408718d8-141d-acc8-2a26-73c097eff0c8 2024-11-25T03:41:10,720 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/408718d8-141d-acc8-2a26-73c097eff0c8/cluster_86ef855a-7356-d2a4-6f37-d8b759212c31/zookeeper_0, clientPort=56467, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/408718d8-141d-acc8-2a26-73c097eff0c8/cluster_86ef855a-7356-d2a4-6f37-d8b759212c31/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/408718d8-141d-acc8-2a26-73c097eff0c8/cluster_86ef855a-7356-d2a4-6f37-d8b759212c31/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-25T03:41:10,729 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=56467 2024-11-25T03:41:10,743 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T03:41:10,746 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T03:41:10,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741825_1001 (size=7) 2024-11-25T03:41:10,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33637 is added to blk_1073741825_1001 (size=7) 2024-11-25T03:41:10,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41447 is added to blk_1073741825_1001 (size=7) 2024-11-25T03:41:11,385 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921 with version=8 2024-11-25T03:41:11,385 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/hbase-staging 2024-11-25T03:41:11,463 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-25T03:41:11,730 INFO [Time-limited test {}] client.ConnectionUtils(128): master/d73269e94d72:0 server-side Connection retries=45 2024-11-25T03:41:11,738 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T03:41:11,739 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T03:41:11,744 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T03:41:11,744 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T03:41:11,744 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T03:41:11,900 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-25T03:41:11,961 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-25T03:41:11,970 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-25T03:41:11,974 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T03:41:11,997 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 3938 (auto-detected) 2024-11-25T03:41:11,998 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-25T03:41:12,016 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40325 2024-11-25T03:41:12,036 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:40325 connecting to ZooKeeper ensemble=127.0.0.1:56467 2024-11-25T03:41:12,154 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:403250x0, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T03:41:12,157 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:40325-0x1017014aa590000 connected 2024-11-25T03:41:12,252 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T03:41:12,255 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T03:41:12,264 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40325-0x1017014aa590000, quorum=127.0.0.1:56467, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T03:41:12,268 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921, hbase.cluster.distributed=false 2024-11-25T03:41:12,293 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40325-0x1017014aa590000, quorum=127.0.0.1:56467, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-25T03:41:12,298 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40325 2024-11-25T03:41:12,298 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40325 2024-11-25T03:41:12,299 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40325 2024-11-25T03:41:12,299 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40325 2024-11-25T03:41:12,300 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40325 2024-11-25T03:41:12,405 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/d73269e94d72:0 server-side Connection retries=45 2024-11-25T03:41:12,407 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T03:41:12,407 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T03:41:12,407 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T03:41:12,408 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T03:41:12,408 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T03:41:12,410 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-25T03:41:12,413 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T03:41:12,414 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35981 2024-11-25T03:41:12,417 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35981 connecting to ZooKeeper ensemble=127.0.0.1:56467 2024-11-25T03:41:12,418 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T03:41:12,425 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T03:41:12,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:359810x0, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T03:41:12,441 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:359810x0, quorum=127.0.0.1:56467, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T03:41:12,441 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35981-0x1017014aa590001 connected 2024-11-25T03:41:12,445 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-25T03:41:12,452 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-25T03:41:12,455 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35981-0x1017014aa590001, quorum=127.0.0.1:56467, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-25T03:41:12,459 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35981-0x1017014aa590001, quorum=127.0.0.1:56467, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-25T03:41:12,460 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35981 2024-11-25T03:41:12,460 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35981 2024-11-25T03:41:12,461 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35981 2024-11-25T03:41:12,461 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35981 2024-11-25T03:41:12,461 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35981 2024-11-25T03:41:12,477 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/d73269e94d72:0 server-side Connection retries=45 2024-11-25T03:41:12,477 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T03:41:12,478 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T03:41:12,478 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T03:41:12,478 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T03:41:12,478 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T03:41:12,478 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-25T03:41:12,479 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T03:41:12,479 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33145 2024-11-25T03:41:12,481 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33145 connecting to ZooKeeper ensemble=127.0.0.1:56467 2024-11-25T03:41:12,482 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T03:41:12,485 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T03:41:12,501 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:331450x0, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T03:41:12,502 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33145-0x1017014aa590002, quorum=127.0.0.1:56467, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T03:41:12,502 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33145-0x1017014aa590002 connected 2024-11-25T03:41:12,502 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-25T03:41:12,503 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-25T03:41:12,504 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33145-0x1017014aa590002, quorum=127.0.0.1:56467, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-25T03:41:12,506 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33145-0x1017014aa590002, quorum=127.0.0.1:56467, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-25T03:41:12,508 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33145 2024-11-25T03:41:12,511 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33145 2024-11-25T03:41:12,511 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33145 2024-11-25T03:41:12,512 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33145 2024-11-25T03:41:12,512 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33145 2024-11-25T03:41:12,529 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/d73269e94d72:0 server-side Connection retries=45 2024-11-25T03:41:12,529 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T03:41:12,529 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T03:41:12,529 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T03:41:12,529 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T03:41:12,530 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T03:41:12,530 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-25T03:41:12,530 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T03:41:12,532 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38397 2024-11-25T03:41:12,534 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38397 connecting to ZooKeeper ensemble=127.0.0.1:56467 2024-11-25T03:41:12,535 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T03:41:12,537 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T03:41:12,553 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:383970x0, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T03:41:12,554 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:383970x0, quorum=127.0.0.1:56467, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T03:41:12,554 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38397-0x1017014aa590003 connected 2024-11-25T03:41:12,555 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-25T03:41:12,555 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-25T03:41:12,556 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38397-0x1017014aa590003, quorum=127.0.0.1:56467, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-25T03:41:12,558 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38397-0x1017014aa590003, quorum=127.0.0.1:56467, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-25T03:41:12,559 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38397 2024-11-25T03:41:12,560 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38397 2024-11-25T03:41:12,560 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38397 2024-11-25T03:41:12,561 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38397 2024-11-25T03:41:12,561 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38397 2024-11-25T03:41:12,576 DEBUG [M:0;d73269e94d72:40325 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;d73269e94d72:40325 2024-11-25T03:41:12,577 INFO [master/d73269e94d72:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/d73269e94d72,40325,1732506071573 2024-11-25T03:41:12,588 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35981-0x1017014aa590001, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T03:41:12,588 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33145-0x1017014aa590002, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T03:41:12,588 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38397-0x1017014aa590003, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T03:41:12,588 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40325-0x1017014aa590000, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T03:41:12,590 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40325-0x1017014aa590000, quorum=127.0.0.1:56467, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/d73269e94d72,40325,1732506071573 2024-11-25T03:41:12,619 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33145-0x1017014aa590002, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-25T03:41:12,619 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38397-0x1017014aa590003, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-25T03:41:12,619 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40325-0x1017014aa590000, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:41:12,619 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35981-0x1017014aa590001, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-25T03:41:12,619 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33145-0x1017014aa590002, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:41:12,619 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35981-0x1017014aa590001, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:41:12,619 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38397-0x1017014aa590003, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:41:12,620 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40325-0x1017014aa590000, quorum=127.0.0.1:56467, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-25T03:41:12,622 INFO [master/d73269e94d72:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/d73269e94d72,40325,1732506071573 from backup master directory 2024-11-25T03:41:12,630 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35981-0x1017014aa590001, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T03:41:12,630 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33145-0x1017014aa590002, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T03:41:12,630 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38397-0x1017014aa590003, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T03:41:12,630 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40325-0x1017014aa590000, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/d73269e94d72,40325,1732506071573 2024-11-25T03:41:12,630 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40325-0x1017014aa590000, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T03:41:12,631 WARN [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T03:41:12,631 INFO [master/d73269e94d72:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=d73269e94d72,40325,1732506071573 2024-11-25T03:41:12,634 INFO [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-25T03:41:12,635 INFO [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-25T03:41:12,688 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/hbase.id] with ID: d383aa6f-9d14-49f9-a255-37adba8d78f1 2024-11-25T03:41:12,688 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/.tmp/hbase.id 2024-11-25T03:41:12,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741826_1002 (size=42) 2024-11-25T03:41:12,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33637 is added to blk_1073741826_1002 (size=42) 2024-11-25T03:41:12,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41447 is added to blk_1073741826_1002 (size=42) 2024-11-25T03:41:12,702 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/.tmp/hbase.id]:[hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/hbase.id] 2024-11-25T03:41:12,748 INFO [master/d73269e94d72:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T03:41:12,753 INFO [master/d73269e94d72:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-25T03:41:12,772 INFO [master/d73269e94d72:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-25T03:41:12,785 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38397-0x1017014aa590003, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:41:12,785 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35981-0x1017014aa590001, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:41:12,785 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40325-0x1017014aa590000, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:41:12,785 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33145-0x1017014aa590002, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:41:12,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41447 is added to blk_1073741827_1003 (size=196) 2024-11-25T03:41:12,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33637 is added to blk_1073741827_1003 (size=196) 2024-11-25T03:41:12,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741827_1003 (size=196) 2024-11-25T03:41:12,822 INFO [master/d73269e94d72:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T03:41:12,824 INFO [master/d73269e94d72:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-25T03:41:12,836 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:41:12,840 INFO [master/d73269e94d72:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T03:41:12,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741828_1004 (size=1189) 2024-11-25T03:41:12,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41447 is added to blk_1073741828_1004 (size=1189) 2024-11-25T03:41:12,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33637 is added to blk_1073741828_1004 (size=1189) 2024-11-25T03:41:12,886 INFO [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/MasterData/data/master/store 2024-11-25T03:41:12,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41447 is added to blk_1073741829_1005 (size=34) 2024-11-25T03:41:12,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33637 is added to blk_1073741829_1005 (size=34) 2024-11-25T03:41:12,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741829_1005 (size=34) 2024-11-25T03:41:12,917 INFO [master/d73269e94d72:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-25T03:41:12,921 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:41:12,923 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-25T03:41:12,923 INFO [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T03:41:12,924 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T03:41:12,926 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-25T03:41:12,926 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T03:41:12,926 INFO [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T03:41:12,928 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732506072923Disabling compacts and flushes for region at 1732506072923Disabling writes for close at 1732506072926 (+3 ms)Writing region close event to WAL at 1732506072926Closed at 1732506072926 2024-11-25T03:41:12,930 WARN [master/d73269e94d72:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/MasterData/data/master/store/.initializing 2024-11-25T03:41:12,931 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/MasterData/WALs/d73269e94d72,40325,1732506071573 2024-11-25T03:41:12,941 INFO [master/d73269e94d72:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T03:41:12,957 INFO [master/d73269e94d72:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d73269e94d72%2C40325%2C1732506071573, suffix=, logDir=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/MasterData/WALs/d73269e94d72,40325,1732506071573, archiveDir=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/MasterData/oldWALs, maxLogs=10 2024-11-25T03:41:12,979 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/MasterData/WALs/d73269e94d72,40325,1732506071573/d73269e94d72%2C40325%2C1732506071573.1732506072961, exclude list is [], retry=0 2024-11-25T03:41:12,996 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33637,DS-88ab84a9-630b-478a-9aeb-f2157b7da108,DISK] 2024-11-25T03:41:12,996 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41447,DS-43ff8d69-2bf0-4a76-b42b-90f4bc41dccf,DISK] 2024-11-25T03:41:12,996 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34311,DS-5d64405e-cf5f-4842-9c54-201a1e1e22d8,DISK] 2024-11-25T03:41:12,999 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-25T03:41:13,037 INFO [master/d73269e94d72:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/MasterData/WALs/d73269e94d72,40325,1732506071573/d73269e94d72%2C40325%2C1732506071573.1732506072961 2024-11-25T03:41:13,038 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41877:41877),(127.0.0.1/127.0.0.1:35677:35677),(127.0.0.1/127.0.0.1:44951:44951)] 2024-11-25T03:41:13,039 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:41:13,039 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:41:13,042 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:41:13,044 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:41:13,082 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:41:13,104 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-25T03:41:13,107 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:41:13,110 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:41:13,110 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:41:13,114 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-25T03:41:13,115 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:41:13,116 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:41:13,116 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:41:13,119 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-25T03:41:13,120 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:41:13,121 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:41:13,121 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:41:13,124 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-25T03:41:13,125 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:41:13,126 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:41:13,126 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:41:13,130 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:41:13,132 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:41:13,137 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:41:13,138 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:41:13,141 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-25T03:41:13,146 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:41:13,151 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:41:13,153 INFO [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63627381, jitterRate=-0.05187813937664032}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-25T03:41:13,161 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732506073059Initializing all the Stores at 1732506073062 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732506073062Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506073063 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506073063Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506073064 (+1 ms)Cleaning up temporary data from old regions at 1732506073138 (+74 ms)Region opened successfully at 1732506073161 (+23 ms) 2024-11-25T03:41:13,163 INFO [master/d73269e94d72:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-25T03:41:13,196 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@118fd959, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d73269e94d72/172.17.0.2:0 2024-11-25T03:41:13,225 INFO [master/d73269e94d72:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-25T03:41:13,236 INFO [master/d73269e94d72:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-25T03:41:13,237 INFO [master/d73269e94d72:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-25T03:41:13,240 INFO [master/d73269e94d72:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-25T03:41:13,241 INFO [master/d73269e94d72:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-25T03:41:13,247 INFO [master/d73269e94d72:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-25T03:41:13,247 INFO [master/d73269e94d72:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-25T03:41:13,275 INFO [master/d73269e94d72:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-25T03:41:13,283 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40325-0x1017014aa590000, quorum=127.0.0.1:56467, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-25T03:41:13,324 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-25T03:41:13,328 INFO [master/d73269e94d72:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-25T03:41:13,331 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40325-0x1017014aa590000, quorum=127.0.0.1:56467, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-25T03:41:13,342 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-25T03:41:13,344 INFO [master/d73269e94d72:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-25T03:41:13,348 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40325-0x1017014aa590000, quorum=127.0.0.1:56467, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-25T03:41:13,356 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-25T03:41:13,357 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40325-0x1017014aa590000, quorum=127.0.0.1:56467, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-25T03:41:13,366 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-25T03:41:13,386 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40325-0x1017014aa590000, quorum=127.0.0.1:56467, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-25T03:41:13,398 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-25T03:41:13,409 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33145-0x1017014aa590002, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-25T03:41:13,409 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40325-0x1017014aa590000, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-25T03:41:13,409 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35981-0x1017014aa590001, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-25T03:41:13,409 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40325-0x1017014aa590000, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:41:13,409 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33145-0x1017014aa590002, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:41:13,409 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35981-0x1017014aa590001, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:41:13,409 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38397-0x1017014aa590003, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-25T03:41:13,409 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38397-0x1017014aa590003, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:41:13,413 INFO [master/d73269e94d72:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=d73269e94d72,40325,1732506071573, sessionid=0x1017014aa590000, setting cluster-up flag (Was=false) 2024-11-25T03:41:13,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40325-0x1017014aa590000, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:41:13,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35981-0x1017014aa590001, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:41:13,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38397-0x1017014aa590003, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:41:13,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33145-0x1017014aa590002, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:41:13,472 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-25T03:41:13,477 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d73269e94d72,40325,1732506071573 2024-11-25T03:41:13,504 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40325-0x1017014aa590000, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:41:13,504 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35981-0x1017014aa590001, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:41:13,504 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33145-0x1017014aa590002, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:41:13,504 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38397-0x1017014aa590003, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:41:13,535 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-25T03:41:13,538 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d73269e94d72,40325,1732506071573 2024-11-25T03:41:13,549 INFO [master/d73269e94d72:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-25T03:41:13,565 INFO [RS:1;d73269e94d72:33145 {}] regionserver.HRegionServer(746): ClusterId : d383aa6f-9d14-49f9-a255-37adba8d78f1 2024-11-25T03:41:13,565 INFO [RS:2;d73269e94d72:38397 {}] regionserver.HRegionServer(746): ClusterId : d383aa6f-9d14-49f9-a255-37adba8d78f1 2024-11-25T03:41:13,565 INFO [RS:0;d73269e94d72:35981 {}] regionserver.HRegionServer(746): ClusterId : d383aa6f-9d14-49f9-a255-37adba8d78f1 2024-11-25T03:41:13,567 DEBUG [RS:1;d73269e94d72:33145 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-25T03:41:13,567 DEBUG [RS:2;d73269e94d72:38397 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-25T03:41:13,567 DEBUG [RS:0;d73269e94d72:35981 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-25T03:41:13,579 DEBUG [RS:1;d73269e94d72:33145 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-25T03:41:13,579 DEBUG [RS:0;d73269e94d72:35981 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-25T03:41:13,579 DEBUG [RS:2;d73269e94d72:38397 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-25T03:41:13,579 DEBUG [RS:1;d73269e94d72:33145 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-25T03:41:13,579 DEBUG [RS:0;d73269e94d72:35981 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-25T03:41:13,579 DEBUG [RS:2;d73269e94d72:38397 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-25T03:41:13,589 DEBUG [RS:2;d73269e94d72:38397 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-25T03:41:13,589 DEBUG [RS:0;d73269e94d72:35981 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-25T03:41:13,589 DEBUG [RS:1;d73269e94d72:33145 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-25T03:41:13,589 DEBUG [RS:2;d73269e94d72:38397 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5eda6f92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d73269e94d72/172.17.0.2:0 2024-11-25T03:41:13,589 DEBUG [RS:1;d73269e94d72:33145 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ba52322, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d73269e94d72/172.17.0.2:0 2024-11-25T03:41:13,589 DEBUG [RS:0;d73269e94d72:35981 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d355151, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d73269e94d72/172.17.0.2:0 2024-11-25T03:41:13,607 DEBUG [RS:2;d73269e94d72:38397 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;d73269e94d72:38397 2024-11-25T03:41:13,607 DEBUG [RS:0;d73269e94d72:35981 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;d73269e94d72:35981 2024-11-25T03:41:13,607 DEBUG [RS:1;d73269e94d72:33145 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;d73269e94d72:33145 2024-11-25T03:41:13,611 INFO [RS:1;d73269e94d72:33145 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-25T03:41:13,611 INFO [RS:2;d73269e94d72:38397 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-25T03:41:13,611 INFO [RS:0;d73269e94d72:35981 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-25T03:41:13,611 INFO [RS:1;d73269e94d72:33145 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-25T03:41:13,611 INFO [RS:0;d73269e94d72:35981 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-25T03:41:13,611 INFO [RS:2;d73269e94d72:38397 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-25T03:41:13,611 DEBUG [RS:1;d73269e94d72:33145 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-25T03:41:13,611 DEBUG [RS:0;d73269e94d72:35981 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-25T03:41:13,611 DEBUG [RS:2;d73269e94d72:38397 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-25T03:41:13,614 INFO [RS:1;d73269e94d72:33145 {}] regionserver.HRegionServer(2659): reportForDuty to master=d73269e94d72,40325,1732506071573 with port=33145, startcode=1732506072477 2024-11-25T03:41:13,614 INFO [RS:0;d73269e94d72:35981 {}] regionserver.HRegionServer(2659): reportForDuty to master=d73269e94d72,40325,1732506071573 with port=35981, startcode=1732506072374 2024-11-25T03:41:13,614 INFO [RS:2;d73269e94d72:38397 {}] regionserver.HRegionServer(2659): reportForDuty to master=d73269e94d72,40325,1732506071573 with port=38397, startcode=1732506072528 2024-11-25T03:41:13,625 DEBUG [RS:2;d73269e94d72:38397 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-25T03:41:13,625 DEBUG [RS:0;d73269e94d72:35981 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-25T03:41:13,625 DEBUG [RS:1;d73269e94d72:33145 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-25T03:41:13,625 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-25T03:41:13,635 INFO [master/d73269e94d72:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-25T03:41:13,641 INFO [master/d73269e94d72:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-25T03:41:13,647 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: d73269e94d72,40325,1732506071573 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-25T03:41:13,654 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/d73269e94d72:0, corePoolSize=5, maxPoolSize=5 2024-11-25T03:41:13,655 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/d73269e94d72:0, corePoolSize=5, maxPoolSize=5 2024-11-25T03:41:13,655 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/d73269e94d72:0, corePoolSize=5, maxPoolSize=5 2024-11-25T03:41:13,655 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/d73269e94d72:0, corePoolSize=5, maxPoolSize=5 2024-11-25T03:41:13,655 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/d73269e94d72:0, corePoolSize=10, maxPoolSize=10 2024-11-25T03:41:13,655 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:41:13,655 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/d73269e94d72:0, corePoolSize=2, maxPoolSize=2 2024-11-25T03:41:13,655 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:41:13,656 INFO [master/d73269e94d72:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732506103656 2024-11-25T03:41:13,658 INFO [master/d73269e94d72:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-25T03:41:13,660 INFO [master/d73269e94d72:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-25T03:41:13,660 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59713, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-25T03:41:13,661 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T03:41:13,661 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-25T03:41:13,661 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59199, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-25T03:41:13,662 INFO [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57571, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-25T03:41:13,664 INFO [master/d73269e94d72:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-25T03:41:13,664 INFO [master/d73269e94d72:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-25T03:41:13,664 INFO [master/d73269e94d72:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-25T03:41:13,664 INFO [master/d73269e94d72:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-25T03:41:13,665 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-25T03:41:13,667 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:41:13,667 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-25T03:41:13,669 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40325 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T03:41:13,669 INFO [master/d73269e94d72:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-25T03:41:13,671 INFO [master/d73269e94d72:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-25T03:41:13,671 INFO [master/d73269e94d72:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-25T03:41:13,673 INFO [master/d73269e94d72:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-25T03:41:13,673 INFO [master/d73269e94d72:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-25T03:41:13,675 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40325 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T03:41:13,676 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/d73269e94d72:0:becomeActiveMaster-HFileCleaner.large.0-1732506073674,5,FailOnTimeoutGroup] 2024-11-25T03:41:13,676 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40325 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T03:41:13,677 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/d73269e94d72:0:becomeActiveMaster-HFileCleaner.small.0-1732506073676,5,FailOnTimeoutGroup] 2024-11-25T03:41:13,678 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-25T03:41:13,678 INFO [master/d73269e94d72:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-25T03:41:13,679 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-25T03:41:13,680 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-25T03:41:13,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741831_1007 (size=1321) 2024-11-25T03:41:13,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41447 is added to blk_1073741831_1007 (size=1321) 2024-11-25T03:41:13,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33637 is added to blk_1073741831_1007 (size=1321) 2024-11-25T03:41:13,726 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-25T03:41:13,726 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921 2024-11-25T03:41:13,733 DEBUG [RS:2;d73269e94d72:38397 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-25T03:41:13,733 DEBUG [RS:1;d73269e94d72:33145 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-25T03:41:13,733 DEBUG [RS:0;d73269e94d72:35981 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-25T03:41:13,733 WARN [RS:0;d73269e94d72:35981 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-25T03:41:13,733 WARN [RS:1;d73269e94d72:33145 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-25T03:41:13,733 WARN [RS:2;d73269e94d72:38397 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-25T03:41:13,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41447 is added to blk_1073741832_1008 (size=32) 2024-11-25T03:41:13,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33637 is added to blk_1073741832_1008 (size=32) 2024-11-25T03:41:13,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741832_1008 (size=32) 2024-11-25T03:41:13,743 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:41:13,745 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-25T03:41:13,748 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-25T03:41:13,748 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:41:13,749 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:41:13,749 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-25T03:41:13,752 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-25T03:41:13,752 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:41:13,753 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:41:13,753 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-25T03:41:13,756 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-25T03:41:13,757 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:41:13,758 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:41:13,758 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-25T03:41:13,761 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-25T03:41:13,761 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:41:13,762 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:41:13,762 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-25T03:41:13,763 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/hbase/meta/1588230740 2024-11-25T03:41:13,764 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/hbase/meta/1588230740 2024-11-25T03:41:13,767 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-25T03:41:13,767 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-25T03:41:13,768 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-25T03:41:13,770 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-25T03:41:13,774 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:41:13,775 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69579888, jitterRate=0.03682112693786621}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-25T03:41:13,778 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732506073743Initializing all the Stores at 1732506073745 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732506073745Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732506073745Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506073745Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732506073745Cleaning up temporary data from old regions at 1732506073767 (+22 ms)Region opened successfully at 1732506073778 (+11 ms) 2024-11-25T03:41:13,779 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-25T03:41:13,779 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-25T03:41:13,779 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-25T03:41:13,779 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-25T03:41:13,779 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-25T03:41:13,781 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-25T03:41:13,781 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732506073778Disabling compacts and flushes for region at 1732506073778Disabling writes for close at 1732506073779 (+1 ms)Writing region close event to WAL at 1732506073780 (+1 ms)Closed at 1732506073780 2024-11-25T03:41:13,784 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T03:41:13,785 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-25T03:41:13,792 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-25T03:41:13,805 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T03:41:13,808 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-25T03:41:13,834 INFO [RS:2;d73269e94d72:38397 {}] regionserver.HRegionServer(2659): reportForDuty to master=d73269e94d72,40325,1732506071573 with port=38397, startcode=1732506072528 2024-11-25T03:41:13,834 INFO [RS:1;d73269e94d72:33145 {}] regionserver.HRegionServer(2659): reportForDuty to master=d73269e94d72,40325,1732506071573 with port=33145, startcode=1732506072477 2024-11-25T03:41:13,834 INFO [RS:0;d73269e94d72:35981 {}] regionserver.HRegionServer(2659): reportForDuty to master=d73269e94d72,40325,1732506071573 with port=35981, startcode=1732506072374 2024-11-25T03:41:13,836 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40325 {}] master.ServerManager(363): Checking decommissioned status of RegionServer d73269e94d72,33145,1732506072477 2024-11-25T03:41:13,838 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40325 {}] master.ServerManager(517): Registering regionserver=d73269e94d72,33145,1732506072477 2024-11-25T03:41:13,846 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40325 {}] master.ServerManager(363): Checking decommissioned status of RegionServer d73269e94d72,38397,1732506072528 2024-11-25T03:41:13,846 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40325 {}] master.ServerManager(517): Registering regionserver=d73269e94d72,38397,1732506072528 2024-11-25T03:41:13,846 DEBUG [RS:1;d73269e94d72:33145 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921 2024-11-25T03:41:13,846 DEBUG [RS:1;d73269e94d72:33145 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44131 2024-11-25T03:41:13,846 DEBUG [RS:1;d73269e94d72:33145 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-25T03:41:13,849 DEBUG [RS:2;d73269e94d72:38397 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921 2024-11-25T03:41:13,849 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40325 {}] master.ServerManager(363): Checking decommissioned status of RegionServer d73269e94d72,35981,1732506072374 2024-11-25T03:41:13,849 DEBUG [RS:2;d73269e94d72:38397 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44131 2024-11-25T03:41:13,849 DEBUG [RS:2;d73269e94d72:38397 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-25T03:41:13,849 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40325 {}] master.ServerManager(517): Registering regionserver=d73269e94d72,35981,1732506072374 2024-11-25T03:41:13,852 DEBUG [RS:0;d73269e94d72:35981 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921 2024-11-25T03:41:13,852 DEBUG [RS:0;d73269e94d72:35981 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44131 2024-11-25T03:41:13,852 DEBUG [RS:0;d73269e94d72:35981 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-25T03:41:13,861 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40325-0x1017014aa590000, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T03:41:13,899 DEBUG [RS:1;d73269e94d72:33145 {}] zookeeper.ZKUtil(111): regionserver:33145-0x1017014aa590002, quorum=127.0.0.1:56467, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d73269e94d72,33145,1732506072477 2024-11-25T03:41:13,900 WARN [RS:1;d73269e94d72:33145 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T03:41:13,900 DEBUG [RS:0;d73269e94d72:35981 {}] zookeeper.ZKUtil(111): regionserver:35981-0x1017014aa590001, quorum=127.0.0.1:56467, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d73269e94d72,35981,1732506072374 2024-11-25T03:41:13,900 INFO [RS:1;d73269e94d72:33145 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T03:41:13,900 DEBUG [RS:2;d73269e94d72:38397 {}] zookeeper.ZKUtil(111): regionserver:38397-0x1017014aa590003, quorum=127.0.0.1:56467, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d73269e94d72,38397,1732506072528 2024-11-25T03:41:13,900 WARN [RS:0;d73269e94d72:35981 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T03:41:13,900 WARN [RS:2;d73269e94d72:38397 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T03:41:13,900 DEBUG [RS:1;d73269e94d72:33145 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/WALs/d73269e94d72,33145,1732506072477 2024-11-25T03:41:13,900 INFO [RS:0;d73269e94d72:35981 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T03:41:13,900 INFO [RS:2;d73269e94d72:38397 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T03:41:13,900 DEBUG [RS:0;d73269e94d72:35981 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/WALs/d73269e94d72,35981,1732506072374 2024-11-25T03:41:13,900 DEBUG [RS:2;d73269e94d72:38397 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/WALs/d73269e94d72,38397,1732506072528 2024-11-25T03:41:13,902 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d73269e94d72,38397,1732506072528] 2024-11-25T03:41:13,902 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d73269e94d72,35981,1732506072374] 2024-11-25T03:41:13,902 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d73269e94d72,33145,1732506072477] 2024-11-25T03:41:13,924 INFO [RS:2;d73269e94d72:38397 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-25T03:41:13,924 INFO [RS:1;d73269e94d72:33145 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-25T03:41:13,924 INFO [RS:0;d73269e94d72:35981 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-25T03:41:13,939 INFO [RS:0;d73269e94d72:35981 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-25T03:41:13,939 INFO [RS:1;d73269e94d72:33145 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-25T03:41:13,939 INFO [RS:2;d73269e94d72:38397 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-25T03:41:13,944 INFO [RS:0;d73269e94d72:35981 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-25T03:41:13,944 INFO [RS:2;d73269e94d72:38397 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-25T03:41:13,944 INFO [RS:1;d73269e94d72:33145 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-25T03:41:13,944 INFO [RS:0;d73269e94d72:35981 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T03:41:13,944 INFO [RS:2;d73269e94d72:38397 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T03:41:13,944 INFO [RS:1;d73269e94d72:33145 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T03:41:13,945 INFO [RS:2;d73269e94d72:38397 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-25T03:41:13,945 INFO [RS:0;d73269e94d72:35981 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-25T03:41:13,948 INFO [RS:1;d73269e94d72:33145 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-25T03:41:13,950 INFO [RS:0;d73269e94d72:35981 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-25T03:41:13,950 INFO [RS:1;d73269e94d72:33145 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-25T03:41:13,950 INFO [RS:2;d73269e94d72:38397 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-25T03:41:13,952 INFO [RS:0;d73269e94d72:35981 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-25T03:41:13,952 INFO [RS:2;d73269e94d72:38397 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-25T03:41:13,952 INFO [RS:1;d73269e94d72:33145 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-25T03:41:13,953 DEBUG [RS:1;d73269e94d72:33145 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:41:13,953 DEBUG [RS:0;d73269e94d72:35981 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:41:13,953 DEBUG [RS:2;d73269e94d72:38397 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:41:13,953 DEBUG [RS:1;d73269e94d72:33145 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:41:13,953 DEBUG [RS:0;d73269e94d72:35981 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:41:13,953 DEBUG [RS:2;d73269e94d72:38397 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:41:13,953 DEBUG [RS:1;d73269e94d72:33145 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:41:13,953 DEBUG [RS:2;d73269e94d72:38397 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:41:13,953 DEBUG [RS:0;d73269e94d72:35981 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:41:13,953 DEBUG [RS:1;d73269e94d72:33145 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:41:13,953 DEBUG [RS:0;d73269e94d72:35981 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:41:13,953 DEBUG [RS:2;d73269e94d72:38397 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:41:13,953 DEBUG [RS:1;d73269e94d72:33145 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:41:13,953 DEBUG [RS:2;d73269e94d72:38397 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:41:13,953 DEBUG [RS:0;d73269e94d72:35981 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:41:13,953 DEBUG [RS:1;d73269e94d72:33145 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0, corePoolSize=2, maxPoolSize=2 2024-11-25T03:41:13,953 DEBUG [RS:2;d73269e94d72:38397 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0, corePoolSize=2, maxPoolSize=2 2024-11-25T03:41:13,953 DEBUG [RS:0;d73269e94d72:35981 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0, corePoolSize=2, maxPoolSize=2 2024-11-25T03:41:13,954 DEBUG [RS:1;d73269e94d72:33145 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:41:13,954 DEBUG [RS:2;d73269e94d72:38397 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:41:13,954 DEBUG [RS:0;d73269e94d72:35981 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:41:13,954 DEBUG [RS:2;d73269e94d72:38397 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:41:13,954 DEBUG [RS:0;d73269e94d72:35981 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:41:13,954 DEBUG [RS:1;d73269e94d72:33145 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:41:13,954 DEBUG [RS:0;d73269e94d72:35981 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:41:13,954 DEBUG [RS:1;d73269e94d72:33145 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:41:13,954 DEBUG [RS:2;d73269e94d72:38397 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:41:13,954 DEBUG [RS:0;d73269e94d72:35981 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:41:13,954 DEBUG [RS:1;d73269e94d72:33145 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:41:13,954 DEBUG [RS:2;d73269e94d72:38397 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:41:13,954 DEBUG [RS:0;d73269e94d72:35981 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:41:13,954 DEBUG [RS:1;d73269e94d72:33145 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:41:13,954 DEBUG [RS:2;d73269e94d72:38397 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:41:13,954 DEBUG [RS:0;d73269e94d72:35981 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:41:13,954 DEBUG [RS:1;d73269e94d72:33145 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:41:13,954 DEBUG [RS:2;d73269e94d72:38397 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:41:13,955 DEBUG [RS:0;d73269e94d72:35981 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d73269e94d72:0, corePoolSize=3, maxPoolSize=3 2024-11-25T03:41:13,955 DEBUG [RS:1;d73269e94d72:33145 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d73269e94d72:0, corePoolSize=3, maxPoolSize=3 2024-11-25T03:41:13,955 DEBUG [RS:2;d73269e94d72:38397 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d73269e94d72:0, corePoolSize=3, maxPoolSize=3 2024-11-25T03:41:13,955 DEBUG [RS:0;d73269e94d72:35981 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d73269e94d72:0, corePoolSize=3, maxPoolSize=3 2024-11-25T03:41:13,955 DEBUG [RS:2;d73269e94d72:38397 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d73269e94d72:0, corePoolSize=3, maxPoolSize=3 2024-11-25T03:41:13,955 DEBUG [RS:1;d73269e94d72:33145 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d73269e94d72:0, corePoolSize=3, maxPoolSize=3 2024-11-25T03:41:13,956 INFO [RS:0;d73269e94d72:35981 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T03:41:13,956 INFO [RS:1;d73269e94d72:33145 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T03:41:13,956 INFO [RS:0;d73269e94d72:35981 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T03:41:13,956 INFO [RS:1;d73269e94d72:33145 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T03:41:13,956 INFO [RS:1;d73269e94d72:33145 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T03:41:13,956 INFO [RS:0;d73269e94d72:35981 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T03:41:13,956 INFO [RS:0;d73269e94d72:35981 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-25T03:41:13,957 INFO [RS:1;d73269e94d72:33145 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-25T03:41:13,957 INFO [RS:0;d73269e94d72:35981 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-25T03:41:13,957 INFO [RS:0;d73269e94d72:35981 {}] hbase.ChoreService(168): Chore ScheduledChore name=d73269e94d72,35981,1732506072374-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T03:41:13,957 INFO [RS:1;d73269e94d72:33145 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-25T03:41:13,957 INFO [RS:1;d73269e94d72:33145 {}] hbase.ChoreService(168): Chore ScheduledChore name=d73269e94d72,33145,1732506072477-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T03:41:13,957 INFO [RS:2;d73269e94d72:38397 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T03:41:13,957 INFO [RS:2;d73269e94d72:38397 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T03:41:13,958 INFO [RS:2;d73269e94d72:38397 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T03:41:13,958 INFO [RS:2;d73269e94d72:38397 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-25T03:41:13,958 INFO [RS:2;d73269e94d72:38397 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-25T03:41:13,958 INFO [RS:2;d73269e94d72:38397 {}] hbase.ChoreService(168): Chore ScheduledChore name=d73269e94d72,38397,1732506072528-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T03:41:13,959 WARN [d73269e94d72:40325 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:41:13,976 INFO [RS:0;d73269e94d72:35981 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-25T03:41:13,976 INFO [RS:1;d73269e94d72:33145 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-25T03:41:13,977 INFO [RS:2;d73269e94d72:38397 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-25T03:41:13,978 INFO [RS:1;d73269e94d72:33145 {}] hbase.ChoreService(168): Chore ScheduledChore name=d73269e94d72,33145,1732506072477-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T03:41:13,978 INFO [RS:2;d73269e94d72:38397 {}] hbase.ChoreService(168): Chore ScheduledChore name=d73269e94d72,38397,1732506072528-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T03:41:13,978 INFO [RS:0;d73269e94d72:35981 {}] hbase.ChoreService(168): Chore ScheduledChore name=d73269e94d72,35981,1732506072374-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T03:41:13,978 INFO [RS:0;d73269e94d72:35981 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T03:41:13,978 INFO [RS:2;d73269e94d72:38397 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T03:41:13,978 INFO [RS:1;d73269e94d72:33145 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T03:41:13,978 INFO [RS:0;d73269e94d72:35981 {}] regionserver.Replication(171): d73269e94d72,35981,1732506072374 started 2024-11-25T03:41:13,978 INFO [RS:2;d73269e94d72:38397 {}] regionserver.Replication(171): d73269e94d72,38397,1732506072528 started 2024-11-25T03:41:13,978 INFO [RS:1;d73269e94d72:33145 {}] regionserver.Replication(171): d73269e94d72,33145,1732506072477 started 2024-11-25T03:41:13,998 INFO [RS:2;d73269e94d72:38397 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T03:41:13,998 INFO [RS:0;d73269e94d72:35981 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T03:41:13,998 INFO [RS:2;d73269e94d72:38397 {}] regionserver.HRegionServer(1482): Serving as d73269e94d72,38397,1732506072528, RpcServer on d73269e94d72/172.17.0.2:38397, sessionid=0x1017014aa590003 2024-11-25T03:41:13,998 INFO [RS:0;d73269e94d72:35981 {}] regionserver.HRegionServer(1482): Serving as d73269e94d72,35981,1732506072374, RpcServer on d73269e94d72/172.17.0.2:35981, sessionid=0x1017014aa590001 2024-11-25T03:41:13,999 DEBUG [RS:2;d73269e94d72:38397 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-25T03:41:13,999 DEBUG [RS:0;d73269e94d72:35981 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-25T03:41:13,999 INFO [RS:1;d73269e94d72:33145 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T03:41:13,999 DEBUG [RS:0;d73269e94d72:35981 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d73269e94d72,35981,1732506072374 2024-11-25T03:41:13,999 INFO [RS:1;d73269e94d72:33145 {}] regionserver.HRegionServer(1482): Serving as d73269e94d72,33145,1732506072477, RpcServer on d73269e94d72/172.17.0.2:33145, sessionid=0x1017014aa590002 2024-11-25T03:41:13,999 DEBUG [RS:2;d73269e94d72:38397 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d73269e94d72,38397,1732506072528 2024-11-25T03:41:13,999 DEBUG [RS:0;d73269e94d72:35981 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd73269e94d72,35981,1732506072374' 2024-11-25T03:41:13,999 DEBUG [RS:1;d73269e94d72:33145 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-25T03:41:13,999 DEBUG [RS:2;d73269e94d72:38397 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd73269e94d72,38397,1732506072528' 2024-11-25T03:41:13,999 DEBUG [RS:0;d73269e94d72:35981 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-25T03:41:13,999 DEBUG [RS:1;d73269e94d72:33145 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d73269e94d72,33145,1732506072477 2024-11-25T03:41:13,999 DEBUG [RS:2;d73269e94d72:38397 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-25T03:41:13,999 DEBUG [RS:1;d73269e94d72:33145 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd73269e94d72,33145,1732506072477' 2024-11-25T03:41:13,999 DEBUG [RS:1;d73269e94d72:33145 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-25T03:41:14,000 DEBUG [RS:0;d73269e94d72:35981 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-25T03:41:14,000 DEBUG [RS:2;d73269e94d72:38397 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-25T03:41:14,000 DEBUG [RS:1;d73269e94d72:33145 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-25T03:41:14,001 DEBUG [RS:0;d73269e94d72:35981 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-25T03:41:14,001 DEBUG [RS:1;d73269e94d72:33145 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-25T03:41:14,001 DEBUG [RS:0;d73269e94d72:35981 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-25T03:41:14,001 DEBUG [RS:1;d73269e94d72:33145 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-25T03:41:14,001 DEBUG [RS:2;d73269e94d72:38397 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-25T03:41:14,001 DEBUG [RS:2;d73269e94d72:38397 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-25T03:41:14,001 DEBUG [RS:1;d73269e94d72:33145 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d73269e94d72,33145,1732506072477 2024-11-25T03:41:14,001 DEBUG [RS:0;d73269e94d72:35981 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d73269e94d72,35981,1732506072374 2024-11-25T03:41:14,001 DEBUG [RS:2;d73269e94d72:38397 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d73269e94d72,38397,1732506072528 2024-11-25T03:41:14,001 DEBUG [RS:0;d73269e94d72:35981 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd73269e94d72,35981,1732506072374' 2024-11-25T03:41:14,001 DEBUG [RS:1;d73269e94d72:33145 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd73269e94d72,33145,1732506072477' 2024-11-25T03:41:14,001 DEBUG [RS:2;d73269e94d72:38397 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd73269e94d72,38397,1732506072528' 2024-11-25T03:41:14,001 DEBUG [RS:0;d73269e94d72:35981 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-25T03:41:14,001 DEBUG [RS:2;d73269e94d72:38397 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-25T03:41:14,001 DEBUG [RS:1;d73269e94d72:33145 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-25T03:41:14,002 DEBUG [RS:1;d73269e94d72:33145 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-25T03:41:14,002 DEBUG [RS:2;d73269e94d72:38397 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-25T03:41:14,002 DEBUG [RS:0;d73269e94d72:35981 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-25T03:41:14,003 DEBUG [RS:2;d73269e94d72:38397 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-25T03:41:14,003 DEBUG [RS:1;d73269e94d72:33145 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-25T03:41:14,003 INFO [RS:2;d73269e94d72:38397 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-25T03:41:14,003 INFO [RS:1;d73269e94d72:33145 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-25T03:41:14,003 INFO [RS:2;d73269e94d72:38397 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-25T03:41:14,003 INFO [RS:1;d73269e94d72:33145 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-25T03:41:14,003 DEBUG [RS:0;d73269e94d72:35981 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-25T03:41:14,003 INFO [RS:0;d73269e94d72:35981 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-25T03:41:14,003 INFO [RS:0;d73269e94d72:35981 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-25T03:41:14,116 INFO [RS:2;d73269e94d72:38397 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T03:41:14,116 INFO [RS:1;d73269e94d72:33145 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T03:41:14,116 INFO [RS:0;d73269e94d72:35981 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T03:41:14,119 INFO [RS:0;d73269e94d72:35981 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d73269e94d72%2C35981%2C1732506072374, suffix=, logDir=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/WALs/d73269e94d72,35981,1732506072374, archiveDir=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/oldWALs, maxLogs=32 2024-11-25T03:41:14,119 INFO [RS:2;d73269e94d72:38397 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d73269e94d72%2C38397%2C1732506072528, suffix=, logDir=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/WALs/d73269e94d72,38397,1732506072528, archiveDir=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/oldWALs, maxLogs=32 2024-11-25T03:41:14,119 INFO [RS:1;d73269e94d72:33145 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d73269e94d72%2C33145%2C1732506072477, suffix=, logDir=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/WALs/d73269e94d72,33145,1732506072477, archiveDir=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/oldWALs, maxLogs=32 2024-11-25T03:41:14,135 DEBUG [RS:0;d73269e94d72:35981 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/WALs/d73269e94d72,35981,1732506072374/d73269e94d72%2C35981%2C1732506072374.1732506074123, exclude list is [], retry=0 2024-11-25T03:41:14,135 DEBUG [RS:2;d73269e94d72:38397 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/WALs/d73269e94d72,38397,1732506072528/d73269e94d72%2C38397%2C1732506072528.1732506074123, exclude list is [], retry=0 2024-11-25T03:41:14,135 DEBUG [RS:1;d73269e94d72:33145 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/WALs/d73269e94d72,33145,1732506072477/d73269e94d72%2C33145%2C1732506072477.1732506074123, exclude list is [], retry=0 2024-11-25T03:41:14,140 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34311,DS-5d64405e-cf5f-4842-9c54-201a1e1e22d8,DISK] 2024-11-25T03:41:14,140 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34311,DS-5d64405e-cf5f-4842-9c54-201a1e1e22d8,DISK] 2024-11-25T03:41:14,140 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41447,DS-43ff8d69-2bf0-4a76-b42b-90f4bc41dccf,DISK] 2024-11-25T03:41:14,140 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33637,DS-88ab84a9-630b-478a-9aeb-f2157b7da108,DISK] 2024-11-25T03:41:14,141 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33637,DS-88ab84a9-630b-478a-9aeb-f2157b7da108,DISK] 2024-11-25T03:41:14,141 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41447,DS-43ff8d69-2bf0-4a76-b42b-90f4bc41dccf,DISK] 2024-11-25T03:41:14,141 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41447,DS-43ff8d69-2bf0-4a76-b42b-90f4bc41dccf,DISK] 2024-11-25T03:41:14,141 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34311,DS-5d64405e-cf5f-4842-9c54-201a1e1e22d8,DISK] 2024-11-25T03:41:14,142 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33637,DS-88ab84a9-630b-478a-9aeb-f2157b7da108,DISK] 2024-11-25T03:41:14,147 INFO [RS:1;d73269e94d72:33145 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/WALs/d73269e94d72,33145,1732506072477/d73269e94d72%2C33145%2C1732506072477.1732506074123 2024-11-25T03:41:14,147 INFO [RS:0;d73269e94d72:35981 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/WALs/d73269e94d72,35981,1732506072374/d73269e94d72%2C35981%2C1732506072374.1732506074123 2024-11-25T03:41:14,147 INFO [RS:2;d73269e94d72:38397 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/WALs/d73269e94d72,38397,1732506072528/d73269e94d72%2C38397%2C1732506072528.1732506074123 2024-11-25T03:41:14,147 DEBUG [RS:1;d73269e94d72:33145 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41877:41877),(127.0.0.1/127.0.0.1:35677:35677),(127.0.0.1/127.0.0.1:44951:44951)] 2024-11-25T03:41:14,147 DEBUG [RS:0;d73269e94d72:35981 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41877:41877),(127.0.0.1/127.0.0.1:44951:44951),(127.0.0.1/127.0.0.1:35677:35677)] 2024-11-25T03:41:14,148 DEBUG [RS:2;d73269e94d72:38397 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44951:44951),(127.0.0.1/127.0.0.1:41877:41877),(127.0.0.1/127.0.0.1:35677:35677)] 2024-11-25T03:41:14,216 DEBUG [d73269e94d72:40325 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-25T03:41:14,226 DEBUG [d73269e94d72:40325 {}] balancer.BalancerClusterState(204): Hosts are {d73269e94d72=0} racks are {/default-rack=0} 2024-11-25T03:41:14,231 DEBUG [d73269e94d72:40325 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-25T03:41:14,232 DEBUG [d73269e94d72:40325 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-25T03:41:14,232 DEBUG [d73269e94d72:40325 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-25T03:41:14,232 DEBUG [d73269e94d72:40325 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-25T03:41:14,232 DEBUG [d73269e94d72:40325 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-25T03:41:14,232 DEBUG [d73269e94d72:40325 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-25T03:41:14,232 INFO [d73269e94d72:40325 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-25T03:41:14,232 INFO [d73269e94d72:40325 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-25T03:41:14,232 INFO [d73269e94d72:40325 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-25T03:41:14,232 DEBUG [d73269e94d72:40325 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-25T03:41:14,238 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=d73269e94d72,33145,1732506072477 2024-11-25T03:41:14,244 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d73269e94d72,33145,1732506072477, state=OPENING 2024-11-25T03:41:14,293 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-25T03:41:14,303 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35981-0x1017014aa590001, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:41:14,303 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38397-0x1017014aa590003, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:41:14,303 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40325-0x1017014aa590000, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:41:14,303 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33145-0x1017014aa590002, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:41:14,305 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:14,305 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:14,305 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:14,305 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:14,308 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T03:41:14,311 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=d73269e94d72,33145,1732506072477}] 2024-11-25T03:41:14,485 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-25T03:41:14,487 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55805, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-25T03:41:14,499 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-25T03:41:14,499 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T03:41:14,500 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-25T03:41:14,503 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d73269e94d72%2C33145%2C1732506072477.meta, suffix=.meta, logDir=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/WALs/d73269e94d72,33145,1732506072477, archiveDir=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/oldWALs, maxLogs=32 2024-11-25T03:41:14,516 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/WALs/d73269e94d72,33145,1732506072477/d73269e94d72%2C33145%2C1732506072477.meta.1732506074505.meta, exclude list is [], retry=0 2024-11-25T03:41:14,521 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41447,DS-43ff8d69-2bf0-4a76-b42b-90f4bc41dccf,DISK] 2024-11-25T03:41:14,521 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33637,DS-88ab84a9-630b-478a-9aeb-f2157b7da108,DISK] 2024-11-25T03:41:14,521 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34311,DS-5d64405e-cf5f-4842-9c54-201a1e1e22d8,DISK] 2024-11-25T03:41:14,524 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/WALs/d73269e94d72,33145,1732506072477/d73269e94d72%2C33145%2C1732506072477.meta.1732506074505.meta 2024-11-25T03:41:14,524 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44951:44951),(127.0.0.1/127.0.0.1:35677:35677),(127.0.0.1/127.0.0.1:41877:41877)] 2024-11-25T03:41:14,525 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:41:14,526 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-25T03:41:14,528 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-25T03:41:14,533 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-25T03:41:14,536 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-25T03:41:14,537 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:41:14,537 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-25T03:41:14,537 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-25T03:41:14,540 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-25T03:41:14,542 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-25T03:41:14,542 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:41:14,542 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:41:14,543 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-25T03:41:14,544 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-25T03:41:14,544 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:41:14,545 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:41:14,545 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-25T03:41:14,546 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-25T03:41:14,547 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:41:14,547 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:41:14,547 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-25T03:41:14,549 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-25T03:41:14,549 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:41:14,550 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:41:14,550 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-25T03:41:14,551 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/hbase/meta/1588230740 2024-11-25T03:41:14,554 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/hbase/meta/1588230740 2024-11-25T03:41:14,557 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-25T03:41:14,557 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-25T03:41:14,558 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-25T03:41:14,560 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-25T03:41:14,562 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64558934, jitterRate=-0.037996917963027954}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-25T03:41:14,562 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-25T03:41:14,564 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732506074538Writing region info on filesystem at 1732506074538Initializing all the Stores at 1732506074539 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732506074540 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732506074540Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506074540Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732506074540Cleaning up temporary data from old regions at 1732506074557 (+17 ms)Running coprocessor post-open hooks at 1732506074562 (+5 ms)Region opened successfully at 1732506074564 (+2 ms) 2024-11-25T03:41:14,570 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732506074478 2024-11-25T03:41:14,582 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-25T03:41:14,582 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-25T03:41:14,584 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=d73269e94d72,33145,1732506072477 2024-11-25T03:41:14,586 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d73269e94d72,33145,1732506072477, state=OPEN 2024-11-25T03:41:14,650 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35981-0x1017014aa590001, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:14,651 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33145-0x1017014aa590002, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:14,651 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38397-0x1017014aa590003, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:14,651 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40325-0x1017014aa590000, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:14,651 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:14,651 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:14,651 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:14,651 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:14,651 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=d73269e94d72,33145,1732506072477 2024-11-25T03:41:14,660 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-25T03:41:14,660 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d73269e94d72,33145,1732506072477 in 341 msec 2024-11-25T03:41:14,670 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-25T03:41:14,670 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 872 msec 2024-11-25T03:41:14,672 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T03:41:14,672 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-25T03:41:14,692 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:41:14,693 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d73269e94d72,33145,1732506072477, seqNum=-1] 2024-11-25T03:41:14,711 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:41:14,713 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52381, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:41:14,733 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1520 sec 2024-11-25T03:41:14,733 INFO [master/d73269e94d72:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732506074733, completionTime=-1 2024-11-25T03:41:14,736 INFO [master/d73269e94d72:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-25T03:41:14,736 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-25T03:41:14,781 INFO [master/d73269e94d72:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-25T03:41:14,781 INFO [master/d73269e94d72:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732506134781 2024-11-25T03:41:14,781 INFO [master/d73269e94d72:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732506194781 2024-11-25T03:41:14,782 INFO [master/d73269e94d72:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 45 msec 2024-11-25T03:41:14,791 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-25T03:41:14,801 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d73269e94d72,40325,1732506071573-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T03:41:14,802 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d73269e94d72,40325,1732506071573-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T03:41:14,802 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d73269e94d72,40325,1732506071573-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T03:41:14,803 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-d73269e94d72:40325, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T03:41:14,804 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-25T03:41:14,806 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-25T03:41:14,810 DEBUG [master/d73269e94d72:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-25T03:41:14,837 INFO [master/d73269e94d72:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.206sec 2024-11-25T03:41:14,839 INFO [master/d73269e94d72:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-25T03:41:14,840 INFO [master/d73269e94d72:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-25T03:41:14,840 INFO [master/d73269e94d72:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-25T03:41:14,841 INFO [master/d73269e94d72:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-25T03:41:14,841 INFO [master/d73269e94d72:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-25T03:41:14,842 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d73269e94d72,40325,1732506071573-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T03:41:14,842 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d73269e94d72,40325,1732506071573-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-25T03:41:14,847 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-25T03:41:14,847 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-25T03:41:14,848 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d73269e94d72,40325,1732506071573-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T03:41:14,883 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ff0a408, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:41:14,887 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-25T03:41:14,887 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-25T03:41:14,891 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d73269e94d72,40325,-1 for getting cluster id 2024-11-25T03:41:14,894 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:41:14,902 DEBUG [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd383aa6f-9d14-49f9-a255-37adba8d78f1' 2024-11-25T03:41:14,904 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:41:14,904 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d383aa6f-9d14-49f9-a255-37adba8d78f1" 2024-11-25T03:41:14,904 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19feb2cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:41:14,904 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d73269e94d72,40325,-1] 2024-11-25T03:41:14,907 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:41:14,908 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:41:14,909 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33760, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:41:14,912 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6238c979, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:41:14,912 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:41:14,918 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d73269e94d72,33145,1732506072477, seqNum=-1] 2024-11-25T03:41:14,919 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:41:14,921 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56330, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:41:14,938 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=d73269e94d72,40325,1732506071573 2024-11-25T03:41:14,941 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-25T03:41:14,945 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is d73269e94d72,40325,1732506071573 2024-11-25T03:41:14,947 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@14abf874 2024-11-25T03:41:14,947 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-25T03:41:14,949 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33772, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-25T03:41:14,955 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40325 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'test', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T03:41:14,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40325 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test 2024-11-25T03:41:14,966 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T03:41:14,968 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40325 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "test" procId is: 4 2024-11-25T03:41:14,969 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:41:14,972 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T03:41:14,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T03:41:14,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41447 is added to blk_1073741837_1013 (size=358) 2024-11-25T03:41:14,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33637 is added to blk_1073741837_1013 (size=358) 2024-11-25T03:41:14,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741837_1013 (size=358) 2024-11-25T03:41:14,989 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3b442507eaac3b5222a3c4f97ff80208, NAME => 'test,,1732506074951.3b442507eaac3b5222a3c4f97ff80208.', STARTKEY => '', ENDKEY => 'bbb'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921 2024-11-25T03:41:14,989 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 5e3a490d70b339ac73875ea7638f934f, NAME => 'test,ccc,1732506074951.5e3a490d70b339ac73875ea7638f934f.', STARTKEY => 'ccc', ENDKEY => 'ddd'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921 2024-11-25T03:41:14,989 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => c73ee0e3b504288792edda3ec9a26049, NAME => 'test,bbb,1732506074951.c73ee0e3b504288792edda3ec9a26049.', STARTKEY => 'bbb', ENDKEY => 'ccc'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921 2024-11-25T03:41:15,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41447 is added to blk_1073741839_1015 (size=45) 2024-11-25T03:41:15,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33637 is added to blk_1073741839_1015 (size=45) 2024-11-25T03:41:15,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741839_1015 (size=45) 2024-11-25T03:41:15,014 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,bbb,1732506074951.c73ee0e3b504288792edda3ec9a26049.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:41:15,014 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing c73ee0e3b504288792edda3ec9a26049, disabling compactions & flushes 2024-11-25T03:41:15,014 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,bbb,1732506074951.c73ee0e3b504288792edda3ec9a26049. 2024-11-25T03:41:15,014 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1732506074951.c73ee0e3b504288792edda3ec9a26049. 2024-11-25T03:41:15,014 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1732506074951.c73ee0e3b504288792edda3ec9a26049. after waiting 0 ms 2024-11-25T03:41:15,014 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1732506074951.c73ee0e3b504288792edda3ec9a26049. 2024-11-25T03:41:15,014 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,bbb,1732506074951.c73ee0e3b504288792edda3ec9a26049. 2024-11-25T03:41:15,014 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for c73ee0e3b504288792edda3ec9a26049: Waiting for close lock at 1732506075014Disabling compacts and flushes for region at 1732506075014Disabling writes for close at 1732506075014Writing region close event to WAL at 1732506075014Closed at 1732506075014 2024-11-25T03:41:15,015 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 85a5fab26752f98e6966cd407e31db69, NAME => 'test,ddd,1732506074951.85a5fab26752f98e6966cd407e31db69.', STARTKEY => 'ddd', ENDKEY => 'eee'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921 2024-11-25T03:41:15,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741838_1014 (size=45) 2024-11-25T03:41:15,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41447 is added to blk_1073741838_1014 (size=45) 2024-11-25T03:41:15,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33637 is added to blk_1073741838_1014 (size=45) 2024-11-25T03:41:15,020 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ccc,1732506074951.5e3a490d70b339ac73875ea7638f934f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:41:15,021 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 5e3a490d70b339ac73875ea7638f934f, disabling compactions & flushes 2024-11-25T03:41:15,021 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ccc,1732506074951.5e3a490d70b339ac73875ea7638f934f. 2024-11-25T03:41:15,021 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1732506074951.5e3a490d70b339ac73875ea7638f934f. 2024-11-25T03:41:15,021 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1732506074951.5e3a490d70b339ac73875ea7638f934f. after waiting 0 ms 2024-11-25T03:41:15,021 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1732506074951.5e3a490d70b339ac73875ea7638f934f. 2024-11-25T03:41:15,021 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ccc,1732506074951.5e3a490d70b339ac73875ea7638f934f. 2024-11-25T03:41:15,021 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 5e3a490d70b339ac73875ea7638f934f: Waiting for close lock at 1732506075021Disabling compacts and flushes for region at 1732506075021Disabling writes for close at 1732506075021Writing region close event to WAL at 1732506075021Closed at 1732506075021 2024-11-25T03:41:15,021 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 8a7d4164efdaed87f07b6f76a94819b3, NAME => 'test,eee,1732506074951.8a7d4164efdaed87f07b6f76a94819b3.', STARTKEY => 'eee', ENDKEY => 'fff'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921 2024-11-25T03:41:15,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741840_1016 (size=42) 2024-11-25T03:41:15,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41447 is added to blk_1073741840_1016 (size=42) 2024-11-25T03:41:15,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33637 is added to blk_1073741840_1016 (size=42) 2024-11-25T03:41:15,033 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,,1732506074951.3b442507eaac3b5222a3c4f97ff80208.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:41:15,033 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 3b442507eaac3b5222a3c4f97ff80208, disabling compactions & flushes 2024-11-25T03:41:15,033 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,,1732506074951.3b442507eaac3b5222a3c4f97ff80208. 2024-11-25T03:41:15,033 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1732506074951.3b442507eaac3b5222a3c4f97ff80208. 2024-11-25T03:41:15,033 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,,1732506074951.3b442507eaac3b5222a3c4f97ff80208. after waiting 0 ms 2024-11-25T03:41:15,033 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,,1732506074951.3b442507eaac3b5222a3c4f97ff80208. 2024-11-25T03:41:15,033 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,,1732506074951.3b442507eaac3b5222a3c4f97ff80208. 2024-11-25T03:41:15,033 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3b442507eaac3b5222a3c4f97ff80208: Waiting for close lock at 1732506075033Disabling compacts and flushes for region at 1732506075033Disabling writes for close at 1732506075033Writing region close event to WAL at 1732506075033Closed at 1732506075033 2024-11-25T03:41:15,034 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => bba1924bf10ad03aeb55628aeffc2363, NAME => 'test,fff,1732506074951.bba1924bf10ad03aeb55628aeffc2363.', STARTKEY => 'fff', ENDKEY => 'ggg'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921 2024-11-25T03:41:15,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41447 is added to blk_1073741841_1017 (size=45) 2024-11-25T03:41:15,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33637 is added to blk_1073741841_1017 (size=45) 2024-11-25T03:41:15,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741841_1017 (size=45) 2024-11-25T03:41:15,048 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ddd,1732506074951.85a5fab26752f98e6966cd407e31db69.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:41:15,048 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 85a5fab26752f98e6966cd407e31db69, disabling compactions & flushes 2024-11-25T03:41:15,048 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ddd,1732506074951.85a5fab26752f98e6966cd407e31db69. 2024-11-25T03:41:15,048 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1732506074951.85a5fab26752f98e6966cd407e31db69. 2024-11-25T03:41:15,048 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1732506074951.85a5fab26752f98e6966cd407e31db69. after waiting 0 ms 2024-11-25T03:41:15,049 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1732506074951.85a5fab26752f98e6966cd407e31db69. 2024-11-25T03:41:15,049 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ddd,1732506074951.85a5fab26752f98e6966cd407e31db69. 2024-11-25T03:41:15,049 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 85a5fab26752f98e6966cd407e31db69: Waiting for close lock at 1732506075048Disabling compacts and flushes for region at 1732506075048Disabling writes for close at 1732506075048Writing region close event to WAL at 1732506075049 (+1 ms)Closed at 1732506075049 2024-11-25T03:41:15,049 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 857a6b6999e909a9b73e698c69291205, NAME => 'test,ggg,1732506074951.857a6b6999e909a9b73e698c69291205.', STARTKEY => 'ggg', ENDKEY => 'hhh'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921 2024-11-25T03:41:15,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741842_1018 (size=45) 2024-11-25T03:41:15,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41447 is added to blk_1073741842_1018 (size=45) 2024-11-25T03:41:15,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33637 is added to blk_1073741842_1018 (size=45) 2024-11-25T03:41:15,061 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,eee,1732506074951.8a7d4164efdaed87f07b6f76a94819b3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:41:15,061 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 8a7d4164efdaed87f07b6f76a94819b3, disabling compactions & flushes 2024-11-25T03:41:15,061 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,eee,1732506074951.8a7d4164efdaed87f07b6f76a94819b3. 2024-11-25T03:41:15,061 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1732506074951.8a7d4164efdaed87f07b6f76a94819b3. 2024-11-25T03:41:15,061 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,eee,1732506074951.8a7d4164efdaed87f07b6f76a94819b3. after waiting 0 ms 2024-11-25T03:41:15,061 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,eee,1732506074951.8a7d4164efdaed87f07b6f76a94819b3. 2024-11-25T03:41:15,061 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,eee,1732506074951.8a7d4164efdaed87f07b6f76a94819b3. 2024-11-25T03:41:15,061 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 8a7d4164efdaed87f07b6f76a94819b3: Waiting for close lock at 1732506075061Disabling compacts and flushes for region at 1732506075061Disabling writes for close at 1732506075061Writing region close event to WAL at 1732506075061Closed at 1732506075061 2024-11-25T03:41:15,062 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 5e791add66a4fee50be34aa83a6d502a, NAME => 'test,hhh,1732506074951.5e791add66a4fee50be34aa83a6d502a.', STARTKEY => 'hhh', ENDKEY => 'iii'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921 2024-11-25T03:41:15,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41447 is added to blk_1073741843_1019 (size=45) 2024-11-25T03:41:15,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33637 is added to blk_1073741843_1019 (size=45) 2024-11-25T03:41:15,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741843_1019 (size=45) 2024-11-25T03:41:15,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33637 is added to blk_1073741845_1021 (size=45) 2024-11-25T03:41:15,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741844_1020 (size=45) 2024-11-25T03:41:15,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41447 is added to blk_1073741844_1020 (size=45) 2024-11-25T03:41:15,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741845_1021 (size=45) 2024-11-25T03:41:15,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33637 is added to blk_1073741844_1020 (size=45) 2024-11-25T03:41:15,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41447 is added to blk_1073741845_1021 (size=45) 2024-11-25T03:41:15,081 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ggg,1732506074951.857a6b6999e909a9b73e698c69291205.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:41:15,081 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 857a6b6999e909a9b73e698c69291205, disabling compactions & flushes 2024-11-25T03:41:15,081 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ggg,1732506074951.857a6b6999e909a9b73e698c69291205. 2024-11-25T03:41:15,081 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1732506074951.857a6b6999e909a9b73e698c69291205. 2024-11-25T03:41:15,081 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1732506074951.857a6b6999e909a9b73e698c69291205. after waiting 0 ms 2024-11-25T03:41:15,081 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1732506074951.857a6b6999e909a9b73e698c69291205. 2024-11-25T03:41:15,081 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ggg,1732506074951.857a6b6999e909a9b73e698c69291205. 2024-11-25T03:41:15,081 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 857a6b6999e909a9b73e698c69291205: Waiting for close lock at 1732506075081Disabling compacts and flushes for region at 1732506075081Disabling writes for close at 1732506075081Writing region close event to WAL at 1732506075081Closed at 1732506075081 2024-11-25T03:41:15,082 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 4c0b379dd00b53bde73bca0e12f321ce, NAME => 'test,iii,1732506074951.4c0b379dd00b53bde73bca0e12f321ce.', STARTKEY => 'iii', ENDKEY => 'jjj'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921 2024-11-25T03:41:15,082 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,hhh,1732506074951.5e791add66a4fee50be34aa83a6d502a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:41:15,082 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 5e791add66a4fee50be34aa83a6d502a, disabling compactions & flushes 2024-11-25T03:41:15,082 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,hhh,1732506074951.5e791add66a4fee50be34aa83a6d502a. 2024-11-25T03:41:15,082 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1732506074951.5e791add66a4fee50be34aa83a6d502a. 2024-11-25T03:41:15,082 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1732506074951.5e791add66a4fee50be34aa83a6d502a. after waiting 0 ms 2024-11-25T03:41:15,082 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1732506074951.5e791add66a4fee50be34aa83a6d502a. 2024-11-25T03:41:15,082 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,hhh,1732506074951.5e791add66a4fee50be34aa83a6d502a. 2024-11-25T03:41:15,082 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 5e791add66a4fee50be34aa83a6d502a: Waiting for close lock at 1732506075082Disabling compacts and flushes for region at 1732506075082Disabling writes for close at 1732506075082Writing region close event to WAL at 1732506075082Closed at 1732506075082 2024-11-25T03:41:15,083 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => f5f64cb2042c701322ac00e312f65f16, NAME => 'test,jjj,1732506074951.f5f64cb2042c701322ac00e312f65f16.', STARTKEY => 'jjj', ENDKEY => 'kkk'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921 2024-11-25T03:41:15,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T03:41:15,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741846_1022 (size=45) 2024-11-25T03:41:15,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41447 is added to blk_1073741846_1022 (size=45) 2024-11-25T03:41:15,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33637 is added to blk_1073741846_1022 (size=45) 2024-11-25T03:41:15,102 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,iii,1732506074951.4c0b379dd00b53bde73bca0e12f321ce.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:41:15,102 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 4c0b379dd00b53bde73bca0e12f321ce, disabling compactions & flushes 2024-11-25T03:41:15,102 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,iii,1732506074951.4c0b379dd00b53bde73bca0e12f321ce. 2024-11-25T03:41:15,103 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1732506074951.4c0b379dd00b53bde73bca0e12f321ce. 2024-11-25T03:41:15,103 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,iii,1732506074951.4c0b379dd00b53bde73bca0e12f321ce. after waiting 0 ms 2024-11-25T03:41:15,103 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,iii,1732506074951.4c0b379dd00b53bde73bca0e12f321ce. 2024-11-25T03:41:15,103 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,iii,1732506074951.4c0b379dd00b53bde73bca0e12f321ce. 2024-11-25T03:41:15,103 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 4c0b379dd00b53bde73bca0e12f321ce: Waiting for close lock at 1732506075102Disabling compacts and flushes for region at 1732506075102Disabling writes for close at 1732506075103 (+1 ms)Writing region close event to WAL at 1732506075103Closed at 1732506075103 2024-11-25T03:41:15,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33637 is added to blk_1073741847_1023 (size=45) 2024-11-25T03:41:15,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741847_1023 (size=45) 2024-11-25T03:41:15,104 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => a4b8682bf62ac734aa4174eba4ee1ddc, NAME => 'test,kkk,1732506074951.a4b8682bf62ac734aa4174eba4ee1ddc.', STARTKEY => 'kkk', ENDKEY => 'lll'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921 2024-11-25T03:41:15,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41447 is added to blk_1073741847_1023 (size=45) 2024-11-25T03:41:15,105 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,jjj,1732506074951.f5f64cb2042c701322ac00e312f65f16.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:41:15,105 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing f5f64cb2042c701322ac00e312f65f16, disabling compactions & flushes 2024-11-25T03:41:15,105 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,jjj,1732506074951.f5f64cb2042c701322ac00e312f65f16. 2024-11-25T03:41:15,105 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1732506074951.f5f64cb2042c701322ac00e312f65f16. 2024-11-25T03:41:15,106 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1732506074951.f5f64cb2042c701322ac00e312f65f16. after waiting 0 ms 2024-11-25T03:41:15,106 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1732506074951.f5f64cb2042c701322ac00e312f65f16. 2024-11-25T03:41:15,106 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,jjj,1732506074951.f5f64cb2042c701322ac00e312f65f16. 2024-11-25T03:41:15,106 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for f5f64cb2042c701322ac00e312f65f16: Waiting for close lock at 1732506075105Disabling compacts and flushes for region at 1732506075105Disabling writes for close at 1732506075106 (+1 ms)Writing region close event to WAL at 1732506075106Closed at 1732506075106 2024-11-25T03:41:15,107 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => eb9ff3be6d639a626d88387efa9120fd, NAME => 'test,lll,1732506074951.eb9ff3be6d639a626d88387efa9120fd.', STARTKEY => 'lll', ENDKEY => 'mmm'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921 2024-11-25T03:41:15,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741848_1024 (size=45) 2024-11-25T03:41:15,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33637 is added to blk_1073741848_1024 (size=45) 2024-11-25T03:41:15,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41447 is added to blk_1073741848_1024 (size=45) 2024-11-25T03:41:15,130 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,kkk,1732506074951.a4b8682bf62ac734aa4174eba4ee1ddc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:41:15,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33637 is added to blk_1073741849_1025 (size=45) 2024-11-25T03:41:15,130 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing a4b8682bf62ac734aa4174eba4ee1ddc, disabling compactions & flushes 2024-11-25T03:41:15,130 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,kkk,1732506074951.a4b8682bf62ac734aa4174eba4ee1ddc. 2024-11-25T03:41:15,130 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1732506074951.a4b8682bf62ac734aa4174eba4ee1ddc. 2024-11-25T03:41:15,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741849_1025 (size=45) 2024-11-25T03:41:15,130 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1732506074951.a4b8682bf62ac734aa4174eba4ee1ddc. after waiting 0 ms 2024-11-25T03:41:15,131 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1732506074951.a4b8682bf62ac734aa4174eba4ee1ddc. 2024-11-25T03:41:15,131 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,kkk,1732506074951.a4b8682bf62ac734aa4174eba4ee1ddc. 2024-11-25T03:41:15,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41447 is added to blk_1073741849_1025 (size=45) 2024-11-25T03:41:15,131 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for a4b8682bf62ac734aa4174eba4ee1ddc: Waiting for close lock at 1732506075130Disabling compacts and flushes for region at 1732506075130Disabling writes for close at 1732506075130Writing region close event to WAL at 1732506075131 (+1 ms)Closed at 1732506075131 2024-11-25T03:41:15,131 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 0c260efb5b2a8a6137010d439efc8d47, NAME => 'test,mmm,1732506074951.0c260efb5b2a8a6137010d439efc8d47.', STARTKEY => 'mmm', ENDKEY => 'nnn'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921 2024-11-25T03:41:15,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741850_1026 (size=45) 2024-11-25T03:41:15,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33637 is added to blk_1073741850_1026 (size=45) 2024-11-25T03:41:15,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41447 is added to blk_1073741850_1026 (size=45) 2024-11-25T03:41:15,143 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,mmm,1732506074951.0c260efb5b2a8a6137010d439efc8d47.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:41:15,143 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 0c260efb5b2a8a6137010d439efc8d47, disabling compactions & flushes 2024-11-25T03:41:15,143 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,mmm,1732506074951.0c260efb5b2a8a6137010d439efc8d47. 2024-11-25T03:41:15,143 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1732506074951.0c260efb5b2a8a6137010d439efc8d47. 2024-11-25T03:41:15,143 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1732506074951.0c260efb5b2a8a6137010d439efc8d47. after waiting 0 ms 2024-11-25T03:41:15,143 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1732506074951.0c260efb5b2a8a6137010d439efc8d47. 2024-11-25T03:41:15,144 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,mmm,1732506074951.0c260efb5b2a8a6137010d439efc8d47. 2024-11-25T03:41:15,144 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 0c260efb5b2a8a6137010d439efc8d47: Waiting for close lock at 1732506075143Disabling compacts and flushes for region at 1732506075143Disabling writes for close at 1732506075143Writing region close event to WAL at 1732506075143Closed at 1732506075143 2024-11-25T03:41:15,145 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 4705fa4ae38907dd8b9783be2dff29da, NAME => 'test,nnn,1732506074951.4705fa4ae38907dd8b9783be2dff29da.', STARTKEY => 'nnn', ENDKEY => 'ooo'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921 2024-11-25T03:41:15,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741851_1027 (size=45) 2024-11-25T03:41:15,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33637 is added to blk_1073741851_1027 (size=45) 2024-11-25T03:41:15,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41447 is added to blk_1073741851_1027 (size=45) 2024-11-25T03:41:15,157 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,nnn,1732506074951.4705fa4ae38907dd8b9783be2dff29da.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:41:15,157 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 4705fa4ae38907dd8b9783be2dff29da, disabling compactions & flushes 2024-11-25T03:41:15,157 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,nnn,1732506074951.4705fa4ae38907dd8b9783be2dff29da. 2024-11-25T03:41:15,157 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1732506074951.4705fa4ae38907dd8b9783be2dff29da. 2024-11-25T03:41:15,157 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1732506074951.4705fa4ae38907dd8b9783be2dff29da. after waiting 0 ms 2024-11-25T03:41:15,157 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1732506074951.4705fa4ae38907dd8b9783be2dff29da. 2024-11-25T03:41:15,157 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,nnn,1732506074951.4705fa4ae38907dd8b9783be2dff29da. 2024-11-25T03:41:15,157 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 4705fa4ae38907dd8b9783be2dff29da: Waiting for close lock at 1732506075157Disabling compacts and flushes for region at 1732506075157Disabling writes for close at 1732506075157Writing region close event to WAL at 1732506075157Closed at 1732506075157 2024-11-25T03:41:15,158 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 5da71c9c32aa33cf6f067ea6d82ae84f, NAME => 'test,ooo,1732506074951.5da71c9c32aa33cf6f067ea6d82ae84f.', STARTKEY => 'ooo', ENDKEY => 'ppp'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921 2024-11-25T03:41:15,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41447 is added to blk_1073741852_1028 (size=45) 2024-11-25T03:41:15,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741852_1028 (size=45) 2024-11-25T03:41:15,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33637 is added to blk_1073741852_1028 (size=45) 2024-11-25T03:41:15,170 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ooo,1732506074951.5da71c9c32aa33cf6f067ea6d82ae84f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:41:15,170 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 5da71c9c32aa33cf6f067ea6d82ae84f, disabling compactions & flushes 2024-11-25T03:41:15,170 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ooo,1732506074951.5da71c9c32aa33cf6f067ea6d82ae84f. 2024-11-25T03:41:15,170 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1732506074951.5da71c9c32aa33cf6f067ea6d82ae84f. 2024-11-25T03:41:15,170 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1732506074951.5da71c9c32aa33cf6f067ea6d82ae84f. after waiting 0 ms 2024-11-25T03:41:15,170 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1732506074951.5da71c9c32aa33cf6f067ea6d82ae84f. 2024-11-25T03:41:15,170 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ooo,1732506074951.5da71c9c32aa33cf6f067ea6d82ae84f. 2024-11-25T03:41:15,170 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 5da71c9c32aa33cf6f067ea6d82ae84f: Waiting for close lock at 1732506075170Disabling compacts and flushes for region at 1732506075170Disabling writes for close at 1732506075170Writing region close event to WAL at 1732506075170Closed at 1732506075170 2024-11-25T03:41:15,171 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 2fc93e882c2b7c5acdb53e9cf1071dc8, NAME => 'test,ppp,1732506074951.2fc93e882c2b7c5acdb53e9cf1071dc8.', STARTKEY => 'ppp', ENDKEY => 'qqq'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921 2024-11-25T03:41:15,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741853_1029 (size=45) 2024-11-25T03:41:15,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41447 is added to blk_1073741853_1029 (size=45) 2024-11-25T03:41:15,182 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ppp,1732506074951.2fc93e882c2b7c5acdb53e9cf1071dc8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:41:15,182 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 2fc93e882c2b7c5acdb53e9cf1071dc8, disabling compactions & flushes 2024-11-25T03:41:15,182 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ppp,1732506074951.2fc93e882c2b7c5acdb53e9cf1071dc8. 2024-11-25T03:41:15,182 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1732506074951.2fc93e882c2b7c5acdb53e9cf1071dc8. 2024-11-25T03:41:15,182 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1732506074951.2fc93e882c2b7c5acdb53e9cf1071dc8. after waiting 0 ms 2024-11-25T03:41:15,182 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1732506074951.2fc93e882c2b7c5acdb53e9cf1071dc8. 2024-11-25T03:41:15,182 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ppp,1732506074951.2fc93e882c2b7c5acdb53e9cf1071dc8. 2024-11-25T03:41:15,182 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 2fc93e882c2b7c5acdb53e9cf1071dc8: Waiting for close lock at 1732506075182Disabling compacts and flushes for region at 1732506075182Disabling writes for close at 1732506075182Writing region close event to WAL at 1732506075182Closed at 1732506075182 2024-11-25T03:41:15,183 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 922ebf14d68aba40da3a2e99777c777b, NAME => 'test,qqq,1732506074951.922ebf14d68aba40da3a2e99777c777b.', STARTKEY => 'qqq', ENDKEY => 'rrr'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921 2024-11-25T03:41:15,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33637 is added to blk_1073741853_1029 (size=45) 2024-11-25T03:41:15,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41447 is added to blk_1073741854_1030 (size=45) 2024-11-25T03:41:15,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741854_1030 (size=45) 2024-11-25T03:41:15,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33637 is added to blk_1073741854_1030 (size=45) 2024-11-25T03:41:15,199 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,qqq,1732506074951.922ebf14d68aba40da3a2e99777c777b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:41:15,199 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 922ebf14d68aba40da3a2e99777c777b, disabling compactions & flushes 2024-11-25T03:41:15,199 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,qqq,1732506074951.922ebf14d68aba40da3a2e99777c777b. 2024-11-25T03:41:15,199 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1732506074951.922ebf14d68aba40da3a2e99777c777b. 2024-11-25T03:41:15,199 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1732506074951.922ebf14d68aba40da3a2e99777c777b. after waiting 0 ms 2024-11-25T03:41:15,199 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1732506074951.922ebf14d68aba40da3a2e99777c777b. 2024-11-25T03:41:15,199 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,qqq,1732506074951.922ebf14d68aba40da3a2e99777c777b. 2024-11-25T03:41:15,199 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 922ebf14d68aba40da3a2e99777c777b: Waiting for close lock at 1732506075199Disabling compacts and flushes for region at 1732506075199Disabling writes for close at 1732506075199Writing region close event to WAL at 1732506075199Closed at 1732506075199 2024-11-25T03:41:15,200 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 745356a682ec14d9f6e2066d456cab06, NAME => 'test,rrr,1732506074951.745356a682ec14d9f6e2066d456cab06.', STARTKEY => 'rrr', ENDKEY => 'sss'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921 2024-11-25T03:41:15,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741855_1031 (size=45) 2024-11-25T03:41:15,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33637 is added to blk_1073741855_1031 (size=45) 2024-11-25T03:41:15,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41447 is added to blk_1073741855_1031 (size=45) 2024-11-25T03:41:15,212 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,rrr,1732506074951.745356a682ec14d9f6e2066d456cab06.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:41:15,212 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 745356a682ec14d9f6e2066d456cab06, disabling compactions & flushes 2024-11-25T03:41:15,213 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,rrr,1732506074951.745356a682ec14d9f6e2066d456cab06. 2024-11-25T03:41:15,213 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1732506074951.745356a682ec14d9f6e2066d456cab06. 2024-11-25T03:41:15,213 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1732506074951.745356a682ec14d9f6e2066d456cab06. after waiting 0 ms 2024-11-25T03:41:15,213 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1732506074951.745356a682ec14d9f6e2066d456cab06. 2024-11-25T03:41:15,213 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,rrr,1732506074951.745356a682ec14d9f6e2066d456cab06. 2024-11-25T03:41:15,213 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 745356a682ec14d9f6e2066d456cab06: Waiting for close lock at 1732506075212Disabling compacts and flushes for region at 1732506075212Disabling writes for close at 1732506075213 (+1 ms)Writing region close event to WAL at 1732506075213Closed at 1732506075213 2024-11-25T03:41:15,214 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 0698446036b7ae35179ebc9b07274db5, NAME => 'test,sss,1732506074951.0698446036b7ae35179ebc9b07274db5.', STARTKEY => 'sss', ENDKEY => 'ttt'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921 2024-11-25T03:41:15,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741856_1032 (size=45) 2024-11-25T03:41:15,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41447 is added to blk_1073741856_1032 (size=45) 2024-11-25T03:41:15,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33637 is added to blk_1073741856_1032 (size=45) 2024-11-25T03:41:15,225 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,sss,1732506074951.0698446036b7ae35179ebc9b07274db5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:41:15,225 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 0698446036b7ae35179ebc9b07274db5, disabling compactions & flushes 2024-11-25T03:41:15,226 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,sss,1732506074951.0698446036b7ae35179ebc9b07274db5. 2024-11-25T03:41:15,226 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1732506074951.0698446036b7ae35179ebc9b07274db5. 2024-11-25T03:41:15,226 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,sss,1732506074951.0698446036b7ae35179ebc9b07274db5. after waiting 0 ms 2024-11-25T03:41:15,226 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,sss,1732506074951.0698446036b7ae35179ebc9b07274db5. 2024-11-25T03:41:15,226 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,sss,1732506074951.0698446036b7ae35179ebc9b07274db5. 2024-11-25T03:41:15,226 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 0698446036b7ae35179ebc9b07274db5: Waiting for close lock at 1732506075225Disabling compacts and flushes for region at 1732506075225Disabling writes for close at 1732506075226 (+1 ms)Writing region close event to WAL at 1732506075226Closed at 1732506075226 2024-11-25T03:41:15,226 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 8b5ae4b02f22b24f40505bb660957004, NAME => 'test,ttt,1732506074951.8b5ae4b02f22b24f40505bb660957004.', STARTKEY => 'ttt', ENDKEY => 'uuu'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921 2024-11-25T03:41:15,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41447 is added to blk_1073741857_1033 (size=45) 2024-11-25T03:41:15,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741857_1033 (size=45) 2024-11-25T03:41:15,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33637 is added to blk_1073741857_1033 (size=45) 2024-11-25T03:41:15,239 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ttt,1732506074951.8b5ae4b02f22b24f40505bb660957004.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:41:15,239 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 8b5ae4b02f22b24f40505bb660957004, disabling compactions & flushes 2024-11-25T03:41:15,239 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ttt,1732506074951.8b5ae4b02f22b24f40505bb660957004. 2024-11-25T03:41:15,239 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1732506074951.8b5ae4b02f22b24f40505bb660957004. 2024-11-25T03:41:15,239 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1732506074951.8b5ae4b02f22b24f40505bb660957004. after waiting 0 ms 2024-11-25T03:41:15,239 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1732506074951.8b5ae4b02f22b24f40505bb660957004. 2024-11-25T03:41:15,239 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ttt,1732506074951.8b5ae4b02f22b24f40505bb660957004. 2024-11-25T03:41:15,239 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 8b5ae4b02f22b24f40505bb660957004: Waiting for close lock at 1732506075239Disabling compacts and flushes for region at 1732506075239Disabling writes for close at 1732506075239Writing region close event to WAL at 1732506075239Closed at 1732506075239 2024-11-25T03:41:15,240 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 6aed1980a8b63b44c776d0990f923aa7, NAME => 'test,uuu,1732506074951.6aed1980a8b63b44c776d0990f923aa7.', STARTKEY => 'uuu', ENDKEY => 'vvv'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921 2024-11-25T03:41:15,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741858_1034 (size=45) 2024-11-25T03:41:15,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33637 is added to blk_1073741858_1034 (size=45) 2024-11-25T03:41:15,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41447 is added to blk_1073741858_1034 (size=45) 2024-11-25T03:41:15,250 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,uuu,1732506074951.6aed1980a8b63b44c776d0990f923aa7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:41:15,250 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 6aed1980a8b63b44c776d0990f923aa7, disabling compactions & flushes 2024-11-25T03:41:15,250 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,uuu,1732506074951.6aed1980a8b63b44c776d0990f923aa7. 2024-11-25T03:41:15,250 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1732506074951.6aed1980a8b63b44c776d0990f923aa7. 2024-11-25T03:41:15,250 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1732506074951.6aed1980a8b63b44c776d0990f923aa7. after waiting 0 ms 2024-11-25T03:41:15,250 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1732506074951.6aed1980a8b63b44c776d0990f923aa7. 2024-11-25T03:41:15,250 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,uuu,1732506074951.6aed1980a8b63b44c776d0990f923aa7. 2024-11-25T03:41:15,250 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 6aed1980a8b63b44c776d0990f923aa7: Waiting for close lock at 1732506075250Disabling compacts and flushes for region at 1732506075250Disabling writes for close at 1732506075250Writing region close event to WAL at 1732506075250Closed at 1732506075250 2024-11-25T03:41:15,251 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 706facb174cc9547b71f08aa672ddcee, NAME => 'test,vvv,1732506074951.706facb174cc9547b71f08aa672ddcee.', STARTKEY => 'vvv', ENDKEY => 'www'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921 2024-11-25T03:41:15,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33637 is added to blk_1073741859_1035 (size=45) 2024-11-25T03:41:15,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741859_1035 (size=45) 2024-11-25T03:41:15,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41447 is added to blk_1073741859_1035 (size=45) 2024-11-25T03:41:15,262 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,vvv,1732506074951.706facb174cc9547b71f08aa672ddcee.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:41:15,262 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 706facb174cc9547b71f08aa672ddcee, disabling compactions & flushes 2024-11-25T03:41:15,262 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,vvv,1732506074951.706facb174cc9547b71f08aa672ddcee. 2024-11-25T03:41:15,262 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1732506074951.706facb174cc9547b71f08aa672ddcee. 2024-11-25T03:41:15,262 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1732506074951.706facb174cc9547b71f08aa672ddcee. after waiting 0 ms 2024-11-25T03:41:15,262 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1732506074951.706facb174cc9547b71f08aa672ddcee. 2024-11-25T03:41:15,262 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,vvv,1732506074951.706facb174cc9547b71f08aa672ddcee. 2024-11-25T03:41:15,263 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 706facb174cc9547b71f08aa672ddcee: Waiting for close lock at 1732506075262Disabling compacts and flushes for region at 1732506075262Disabling writes for close at 1732506075262Writing region close event to WAL at 1732506075262Closed at 1732506075262 2024-11-25T03:41:15,263 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 2b224c7c31e6ff664ce67ac884f573d7, NAME => 'test,www,1732506074951.2b224c7c31e6ff664ce67ac884f573d7.', STARTKEY => 'www', ENDKEY => 'xxx'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921 2024-11-25T03:41:15,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33637 is added to blk_1073741860_1036 (size=45) 2024-11-25T03:41:15,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41447 is added to blk_1073741860_1036 (size=45) 2024-11-25T03:41:15,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741860_1036 (size=45) 2024-11-25T03:41:15,274 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,www,1732506074951.2b224c7c31e6ff664ce67ac884f573d7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:41:15,274 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 2b224c7c31e6ff664ce67ac884f573d7, disabling compactions & flushes 2024-11-25T03:41:15,275 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,www,1732506074951.2b224c7c31e6ff664ce67ac884f573d7. 2024-11-25T03:41:15,275 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1732506074951.2b224c7c31e6ff664ce67ac884f573d7. 2024-11-25T03:41:15,275 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,www,1732506074951.2b224c7c31e6ff664ce67ac884f573d7. after waiting 0 ms 2024-11-25T03:41:15,275 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,www,1732506074951.2b224c7c31e6ff664ce67ac884f573d7. 2024-11-25T03:41:15,275 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,www,1732506074951.2b224c7c31e6ff664ce67ac884f573d7. 2024-11-25T03:41:15,275 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 2b224c7c31e6ff664ce67ac884f573d7: Waiting for close lock at 1732506075274Disabling compacts and flushes for region at 1732506075274Disabling writes for close at 1732506075275 (+1 ms)Writing region close event to WAL at 1732506075275Closed at 1732506075275 2024-11-25T03:41:15,275 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 5f812dacea03120e7730979dc37e38c6, NAME => 'test,xxx,1732506074951.5f812dacea03120e7730979dc37e38c6.', STARTKEY => 'xxx', ENDKEY => 'yyy'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921 2024-11-25T03:41:15,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33637 is added to blk_1073741861_1037 (size=45) 2024-11-25T03:41:15,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41447 is added to blk_1073741861_1037 (size=45) 2024-11-25T03:41:15,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741861_1037 (size=45) 2024-11-25T03:41:15,286 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,xxx,1732506074951.5f812dacea03120e7730979dc37e38c6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:41:15,286 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 5f812dacea03120e7730979dc37e38c6, disabling compactions & flushes 2024-11-25T03:41:15,286 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,xxx,1732506074951.5f812dacea03120e7730979dc37e38c6. 2024-11-25T03:41:15,286 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1732506074951.5f812dacea03120e7730979dc37e38c6. 2024-11-25T03:41:15,286 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1732506074951.5f812dacea03120e7730979dc37e38c6. after waiting 0 ms 2024-11-25T03:41:15,286 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1732506074951.5f812dacea03120e7730979dc37e38c6. 2024-11-25T03:41:15,286 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,xxx,1732506074951.5f812dacea03120e7730979dc37e38c6. 2024-11-25T03:41:15,286 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 5f812dacea03120e7730979dc37e38c6: Waiting for close lock at 1732506075286Disabling compacts and flushes for region at 1732506075286Disabling writes for close at 1732506075286Writing region close event to WAL at 1732506075286Closed at 1732506075286 2024-11-25T03:41:15,287 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => df71ad9490ccea77dfed73e590672db8, NAME => 'test,yyy,1732506074951.df71ad9490ccea77dfed73e590672db8.', STARTKEY => 'yyy', ENDKEY => 'zzz'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921 2024-11-25T03:41:15,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41447 is added to blk_1073741862_1038 (size=45) 2024-11-25T03:41:15,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33637 is added to blk_1073741862_1038 (size=45) 2024-11-25T03:41:15,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T03:41:15,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741862_1038 (size=45) 2024-11-25T03:41:15,297 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,yyy,1732506074951.df71ad9490ccea77dfed73e590672db8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:41:15,297 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing df71ad9490ccea77dfed73e590672db8, disabling compactions & flushes 2024-11-25T03:41:15,297 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,yyy,1732506074951.df71ad9490ccea77dfed73e590672db8. 2024-11-25T03:41:15,297 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1732506074951.df71ad9490ccea77dfed73e590672db8. 2024-11-25T03:41:15,297 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1732506074951.df71ad9490ccea77dfed73e590672db8. after waiting 0 ms 2024-11-25T03:41:15,297 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1732506074951.df71ad9490ccea77dfed73e590672db8. 2024-11-25T03:41:15,297 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,yyy,1732506074951.df71ad9490ccea77dfed73e590672db8. 2024-11-25T03:41:15,297 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for df71ad9490ccea77dfed73e590672db8: Waiting for close lock at 1732506075297Disabling compacts and flushes for region at 1732506075297Disabling writes for close at 1732506075297Writing region close event to WAL at 1732506075297Closed at 1732506075297 2024-11-25T03:41:15,298 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => c769ca025a015554efcc98619008be96, NAME => 'test,zzz,1732506074951.c769ca025a015554efcc98619008be96.', STARTKEY => 'zzz', ENDKEY => ''}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921 2024-11-25T03:41:15,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41447 is added to blk_1073741863_1039 (size=42) 2024-11-25T03:41:15,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741863_1039 (size=42) 2024-11-25T03:41:15,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33637 is added to blk_1073741863_1039 (size=42) 2024-11-25T03:41:15,309 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,zzz,1732506074951.c769ca025a015554efcc98619008be96.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:41:15,309 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing c769ca025a015554efcc98619008be96, disabling compactions & flushes 2024-11-25T03:41:15,309 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,zzz,1732506074951.c769ca025a015554efcc98619008be96. 2024-11-25T03:41:15,309 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1732506074951.c769ca025a015554efcc98619008be96. 2024-11-25T03:41:15,309 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1732506074951.c769ca025a015554efcc98619008be96. after waiting 0 ms 2024-11-25T03:41:15,309 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1732506074951.c769ca025a015554efcc98619008be96. 2024-11-25T03:41:15,309 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,zzz,1732506074951.c769ca025a015554efcc98619008be96. 2024-11-25T03:41:15,309 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for c769ca025a015554efcc98619008be96: Waiting for close lock at 1732506075309Disabling compacts and flushes for region at 1732506075309Disabling writes for close at 1732506075309Writing region close event to WAL at 1732506075309Closed at 1732506075309 2024-11-25T03:41:15,473 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,fff,1732506074951.bba1924bf10ad03aeb55628aeffc2363.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:41:15,473 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing bba1924bf10ad03aeb55628aeffc2363, disabling compactions & flushes 2024-11-25T03:41:15,473 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,fff,1732506074951.bba1924bf10ad03aeb55628aeffc2363. 2024-11-25T03:41:15,473 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1732506074951.bba1924bf10ad03aeb55628aeffc2363. 2024-11-25T03:41:15,474 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,fff,1732506074951.bba1924bf10ad03aeb55628aeffc2363. after waiting 0 ms 2024-11-25T03:41:15,474 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,fff,1732506074951.bba1924bf10ad03aeb55628aeffc2363. 2024-11-25T03:41:15,474 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,fff,1732506074951.bba1924bf10ad03aeb55628aeffc2363. 2024-11-25T03:41:15,474 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for bba1924bf10ad03aeb55628aeffc2363: Waiting for close lock at 1732506075473Disabling compacts and flushes for region at 1732506075473Disabling writes for close at 1732506075474 (+1 ms)Writing region close event to WAL at 1732506075474Closed at 1732506075474 2024-11-25T03:41:15,531 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,lll,1732506074951.eb9ff3be6d639a626d88387efa9120fd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:41:15,531 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing eb9ff3be6d639a626d88387efa9120fd, disabling compactions & flushes 2024-11-25T03:41:15,531 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,lll,1732506074951.eb9ff3be6d639a626d88387efa9120fd. 2024-11-25T03:41:15,531 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1732506074951.eb9ff3be6d639a626d88387efa9120fd. 2024-11-25T03:41:15,531 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,lll,1732506074951.eb9ff3be6d639a626d88387efa9120fd. after waiting 0 ms 2024-11-25T03:41:15,531 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,lll,1732506074951.eb9ff3be6d639a626d88387efa9120fd. 2024-11-25T03:41:15,531 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,lll,1732506074951.eb9ff3be6d639a626d88387efa9120fd. 2024-11-25T03:41:15,531 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for eb9ff3be6d639a626d88387efa9120fd: Waiting for close lock at 1732506075531Disabling compacts and flushes for region at 1732506075531Disabling writes for close at 1732506075531Writing region close event to WAL at 1732506075531Closed at 1732506075531 2024-11-25T03:41:15,534 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T03:41:15,541 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,bbb,1732506074951.c73ee0e3b504288792edda3ec9a26049.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732506075535"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732506075535"}]},"ts":"1732506075535"} 2024-11-25T03:41:15,541 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ccc,1732506074951.5e3a490d70b339ac73875ea7638f934f.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732506075535"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732506075535"}]},"ts":"1732506075535"} 2024-11-25T03:41:15,542 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,,1732506074951.3b442507eaac3b5222a3c4f97ff80208.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1732506075535"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732506075535"}]},"ts":"1732506075535"} 2024-11-25T03:41:15,542 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ddd,1732506074951.85a5fab26752f98e6966cd407e31db69.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732506075535"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732506075535"}]},"ts":"1732506075535"} 2024-11-25T03:41:15,542 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,eee,1732506074951.8a7d4164efdaed87f07b6f76a94819b3.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732506075535"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732506075535"}]},"ts":"1732506075535"} 2024-11-25T03:41:15,542 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ggg,1732506074951.857a6b6999e909a9b73e698c69291205.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732506075535"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732506075535"}]},"ts":"1732506075535"} 2024-11-25T03:41:15,542 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,hhh,1732506074951.5e791add66a4fee50be34aa83a6d502a.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732506075535"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732506075535"}]},"ts":"1732506075535"} 2024-11-25T03:41:15,543 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,iii,1732506074951.4c0b379dd00b53bde73bca0e12f321ce.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732506075535"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732506075535"}]},"ts":"1732506075535"} 2024-11-25T03:41:15,543 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,jjj,1732506074951.f5f64cb2042c701322ac00e312f65f16.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732506075535"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732506075535"}]},"ts":"1732506075535"} 2024-11-25T03:41:15,543 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,kkk,1732506074951.a4b8682bf62ac734aa4174eba4ee1ddc.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732506075535"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732506075535"}]},"ts":"1732506075535"} 2024-11-25T03:41:15,543 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,mmm,1732506074951.0c260efb5b2a8a6137010d439efc8d47.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732506075535"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732506075535"}]},"ts":"1732506075535"} 2024-11-25T03:41:15,544 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,nnn,1732506074951.4705fa4ae38907dd8b9783be2dff29da.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732506075535"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732506075535"}]},"ts":"1732506075535"} 2024-11-25T03:41:15,544 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ooo,1732506074951.5da71c9c32aa33cf6f067ea6d82ae84f.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732506075535"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732506075535"}]},"ts":"1732506075535"} 2024-11-25T03:41:15,544 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ppp,1732506074951.2fc93e882c2b7c5acdb53e9cf1071dc8.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732506075535"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732506075535"}]},"ts":"1732506075535"} 2024-11-25T03:41:15,544 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,qqq,1732506074951.922ebf14d68aba40da3a2e99777c777b.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732506075535"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732506075535"}]},"ts":"1732506075535"} 2024-11-25T03:41:15,544 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,rrr,1732506074951.745356a682ec14d9f6e2066d456cab06.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732506075535"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732506075535"}]},"ts":"1732506075535"} 2024-11-25T03:41:15,544 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,sss,1732506074951.0698446036b7ae35179ebc9b07274db5.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732506075535"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732506075535"}]},"ts":"1732506075535"} 2024-11-25T03:41:15,545 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ttt,1732506074951.8b5ae4b02f22b24f40505bb660957004.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732506075535"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732506075535"}]},"ts":"1732506075535"} 2024-11-25T03:41:15,545 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,uuu,1732506074951.6aed1980a8b63b44c776d0990f923aa7.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732506075535"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732506075535"}]},"ts":"1732506075535"} 2024-11-25T03:41:15,545 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,vvv,1732506074951.706facb174cc9547b71f08aa672ddcee.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732506075535"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732506075535"}]},"ts":"1732506075535"} 2024-11-25T03:41:15,545 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,www,1732506074951.2b224c7c31e6ff664ce67ac884f573d7.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732506075535"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732506075535"}]},"ts":"1732506075535"} 2024-11-25T03:41:15,545 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,xxx,1732506074951.5f812dacea03120e7730979dc37e38c6.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732506075535"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732506075535"}]},"ts":"1732506075535"} 2024-11-25T03:41:15,545 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,yyy,1732506074951.df71ad9490ccea77dfed73e590672db8.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732506075535"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732506075535"}]},"ts":"1732506075535"} 2024-11-25T03:41:15,545 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,zzz,1732506074951.c769ca025a015554efcc98619008be96.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1732506075535"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732506075535"}]},"ts":"1732506075535"} 2024-11-25T03:41:15,545 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,fff,1732506074951.bba1924bf10ad03aeb55628aeffc2363.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732506075535"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732506075535"}]},"ts":"1732506075535"} 2024-11-25T03:41:15,546 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,lll,1732506074951.eb9ff3be6d639a626d88387efa9120fd.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732506075535"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732506075535"}]},"ts":"1732506075535"} 2024-11-25T03:41:15,587 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 26 regions to meta. 2024-11-25T03:41:15,590 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T03:41:15,592 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732506075590"}]},"ts":"1732506075590"} 2024-11-25T03:41:15,596 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLING in hbase:meta 2024-11-25T03:41:15,597 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {d73269e94d72=0} racks are {/default-rack=0} 2024-11-25T03:41:15,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T03:41:15,613 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-25T03:41:15,613 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-25T03:41:15,613 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-25T03:41:15,613 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-25T03:41:15,613 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-25T03:41:15,613 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-25T03:41:15,613 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-25T03:41:15,613 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-25T03:41:15,613 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-25T03:41:15,613 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-25T03:41:15,617 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c73ee0e3b504288792edda3ec9a26049, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5e3a490d70b339ac73875ea7638f934f, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=85a5fab26752f98e6966cd407e31db69, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8a7d4164efdaed87f07b6f76a94819b3, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=bba1924bf10ad03aeb55628aeffc2363, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5e791add66a4fee50be34aa83a6d502a, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4c0b379dd00b53bde73bca0e12f321ce, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f5f64cb2042c701322ac00e312f65f16, ASSIGN}, {pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a4b8682bf62ac734aa4174eba4ee1ddc, ASSIGN}, {pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=eb9ff3be6d639a626d88387efa9120fd, ASSIGN}, {pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0c260efb5b2a8a6137010d439efc8d47, ASSIGN}, {pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4705fa4ae38907dd8b9783be2dff29da, ASSIGN}, {pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5da71c9c32aa33cf6f067ea6d82ae84f, ASSIGN}, {pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2fc93e882c2b7c5acdb53e9cf1071dc8, ASSIGN}, {pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN}, {pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=745356a682ec14d9f6e2066d456cab06, ASSIGN}, {pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0698446036b7ae35179ebc9b07274db5, ASSIGN}, {pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8b5ae4b02f22b24f40505bb660957004, ASSIGN}, {pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6aed1980a8b63b44c776d0990f923aa7, ASSIGN}, {pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=706facb174cc9547b71f08aa672ddcee, ASSIGN}, {pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2b224c7c31e6ff664ce67ac884f573d7, ASSIGN}, {pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5f812dacea03120e7730979dc37e38c6, ASSIGN}, {pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=df71ad9490ccea77dfed73e590672db8, ASSIGN}, {pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c769ca025a015554efcc98619008be96, ASSIGN}] 2024-11-25T03:41:15,626 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c769ca025a015554efcc98619008be96, ASSIGN 2024-11-25T03:41:15,627 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c769ca025a015554efcc98619008be96, ASSIGN; state=OFFLINE, location=d73269e94d72,35981,1732506072374; forceNewPlan=false, retain=false 2024-11-25T03:41:15,628 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=df71ad9490ccea77dfed73e590672db8, ASSIGN 2024-11-25T03:41:15,630 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=df71ad9490ccea77dfed73e590672db8, ASSIGN; state=OFFLINE, location=d73269e94d72,38397,1732506072528; forceNewPlan=false, retain=false 2024-11-25T03:41:15,631 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5f812dacea03120e7730979dc37e38c6, ASSIGN 2024-11-25T03:41:15,633 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=5f812dacea03120e7730979dc37e38c6, ASSIGN; state=OFFLINE, location=d73269e94d72,33145,1732506072477; forceNewPlan=false, retain=false 2024-11-25T03:41:15,635 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2b224c7c31e6ff664ce67ac884f573d7, ASSIGN 2024-11-25T03:41:15,636 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=2b224c7c31e6ff664ce67ac884f573d7, ASSIGN; state=OFFLINE, location=d73269e94d72,35981,1732506072374; forceNewPlan=false, retain=false 2024-11-25T03:41:15,638 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=706facb174cc9547b71f08aa672ddcee, ASSIGN 2024-11-25T03:41:15,639 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=706facb174cc9547b71f08aa672ddcee, ASSIGN; state=OFFLINE, location=d73269e94d72,38397,1732506072528; forceNewPlan=false, retain=false 2024-11-25T03:41:15,641 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6aed1980a8b63b44c776d0990f923aa7, ASSIGN 2024-11-25T03:41:15,642 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=6aed1980a8b63b44c776d0990f923aa7, ASSIGN; state=OFFLINE, location=d73269e94d72,33145,1732506072477; forceNewPlan=false, retain=false 2024-11-25T03:41:15,644 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8b5ae4b02f22b24f40505bb660957004, ASSIGN 2024-11-25T03:41:15,645 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8b5ae4b02f22b24f40505bb660957004, ASSIGN; state=OFFLINE, location=d73269e94d72,35981,1732506072374; forceNewPlan=false, retain=false 2024-11-25T03:41:15,647 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0698446036b7ae35179ebc9b07274db5, ASSIGN 2024-11-25T03:41:15,648 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=0698446036b7ae35179ebc9b07274db5, ASSIGN; state=OFFLINE, location=d73269e94d72,38397,1732506072528; forceNewPlan=false, retain=false 2024-11-25T03:41:15,649 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=745356a682ec14d9f6e2066d456cab06, ASSIGN 2024-11-25T03:41:15,651 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=745356a682ec14d9f6e2066d456cab06, ASSIGN; state=OFFLINE, location=d73269e94d72,33145,1732506072477; forceNewPlan=false, retain=false 2024-11-25T03:41:15,652 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN 2024-11-25T03:41:15,653 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN; state=OFFLINE, location=d73269e94d72,35981,1732506072374; forceNewPlan=false, retain=false 2024-11-25T03:41:15,654 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2fc93e882c2b7c5acdb53e9cf1071dc8, ASSIGN 2024-11-25T03:41:15,656 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=2fc93e882c2b7c5acdb53e9cf1071dc8, ASSIGN; state=OFFLINE, location=d73269e94d72,38397,1732506072528; forceNewPlan=false, retain=false 2024-11-25T03:41:15,657 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5da71c9c32aa33cf6f067ea6d82ae84f, ASSIGN 2024-11-25T03:41:15,658 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=5da71c9c32aa33cf6f067ea6d82ae84f, ASSIGN; state=OFFLINE, location=d73269e94d72,33145,1732506072477; forceNewPlan=false, retain=false 2024-11-25T03:41:15,660 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4705fa4ae38907dd8b9783be2dff29da, ASSIGN 2024-11-25T03:41:15,661 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4705fa4ae38907dd8b9783be2dff29da, ASSIGN; state=OFFLINE, location=d73269e94d72,35981,1732506072374; forceNewPlan=false, retain=false 2024-11-25T03:41:15,662 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0c260efb5b2a8a6137010d439efc8d47, ASSIGN 2024-11-25T03:41:15,664 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=0c260efb5b2a8a6137010d439efc8d47, ASSIGN; state=OFFLINE, location=d73269e94d72,38397,1732506072528; forceNewPlan=false, retain=false 2024-11-25T03:41:15,666 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=eb9ff3be6d639a626d88387efa9120fd, ASSIGN 2024-11-25T03:41:15,667 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=eb9ff3be6d639a626d88387efa9120fd, ASSIGN; state=OFFLINE, location=d73269e94d72,38397,1732506072528; forceNewPlan=false, retain=false 2024-11-25T03:41:15,669 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a4b8682bf62ac734aa4174eba4ee1ddc, ASSIGN 2024-11-25T03:41:15,670 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a4b8682bf62ac734aa4174eba4ee1ddc, ASSIGN; state=OFFLINE, location=d73269e94d72,33145,1732506072477; forceNewPlan=false, retain=false 2024-11-25T03:41:15,672 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f5f64cb2042c701322ac00e312f65f16, ASSIGN 2024-11-25T03:41:15,673 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f5f64cb2042c701322ac00e312f65f16, ASSIGN; state=OFFLINE, location=d73269e94d72,35981,1732506072374; forceNewPlan=false, retain=false 2024-11-25T03:41:15,674 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4c0b379dd00b53bde73bca0e12f321ce, ASSIGN 2024-11-25T03:41:15,675 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4c0b379dd00b53bde73bca0e12f321ce, ASSIGN; state=OFFLINE, location=d73269e94d72,38397,1732506072528; forceNewPlan=false, retain=false 2024-11-25T03:41:15,677 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5e791add66a4fee50be34aa83a6d502a, ASSIGN 2024-11-25T03:41:15,678 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=5e791add66a4fee50be34aa83a6d502a, ASSIGN; state=OFFLINE, location=d73269e94d72,33145,1732506072477; forceNewPlan=false, retain=false 2024-11-25T03:41:15,679 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN 2024-11-25T03:41:15,680 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN; state=OFFLINE, location=d73269e94d72,35981,1732506072374; forceNewPlan=false, retain=false 2024-11-25T03:41:15,682 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=bba1924bf10ad03aeb55628aeffc2363, ASSIGN 2024-11-25T03:41:15,683 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=bba1924bf10ad03aeb55628aeffc2363, ASSIGN; state=OFFLINE, location=d73269e94d72,33145,1732506072477; forceNewPlan=false, retain=false 2024-11-25T03:41:15,684 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8a7d4164efdaed87f07b6f76a94819b3, ASSIGN 2024-11-25T03:41:15,686 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8a7d4164efdaed87f07b6f76a94819b3, ASSIGN; state=OFFLINE, location=d73269e94d72,38397,1732506072528; forceNewPlan=false, retain=false 2024-11-25T03:41:15,687 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=85a5fab26752f98e6966cd407e31db69, ASSIGN 2024-11-25T03:41:15,688 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=85a5fab26752f98e6966cd407e31db69, ASSIGN; state=OFFLINE, location=d73269e94d72,33145,1732506072477; forceNewPlan=false, retain=false 2024-11-25T03:41:15,690 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5e3a490d70b339ac73875ea7638f934f, ASSIGN 2024-11-25T03:41:15,691 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=5e3a490d70b339ac73875ea7638f934f, ASSIGN; state=OFFLINE, location=d73269e94d72,38397,1732506072528; forceNewPlan=false, retain=false 2024-11-25T03:41:15,692 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c73ee0e3b504288792edda3ec9a26049, ASSIGN 2024-11-25T03:41:15,694 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c73ee0e3b504288792edda3ec9a26049, ASSIGN; state=OFFLINE, location=d73269e94d72,33145,1732506072477; forceNewPlan=false, retain=false 2024-11-25T03:41:15,696 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN 2024-11-25T03:41:15,697 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN; state=OFFLINE, location=d73269e94d72,35981,1732506072374; forceNewPlan=false, retain=false 2024-11-25T03:41:15,781 INFO [d73269e94d72:40325 {}] balancer.BaseLoadBalancer(388): Reassigned 26 regions. 26 retained the pre-restart assignment. 2024-11-25T03:41:15,782 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=df71ad9490ccea77dfed73e590672db8, regionState=OPENING, regionLocation=d73269e94d72,38397,1732506072528 2024-11-25T03:41:15,783 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=706facb174cc9547b71f08aa672ddcee, regionState=OPENING, regionLocation=d73269e94d72,38397,1732506072528 2024-11-25T03:41:15,784 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=0698446036b7ae35179ebc9b07274db5, regionState=OPENING, regionLocation=d73269e94d72,38397,1732506072528 2024-11-25T03:41:15,785 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=2fc93e882c2b7c5acdb53e9cf1071dc8, regionState=OPENING, regionLocation=d73269e94d72,38397,1732506072528 2024-11-25T03:41:15,786 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=0c260efb5b2a8a6137010d439efc8d47, regionState=OPENING, regionLocation=d73269e94d72,38397,1732506072528 2024-11-25T03:41:15,786 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=df71ad9490ccea77dfed73e590672db8, ASSIGN because future has completed 2024-11-25T03:41:15,787 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=eb9ff3be6d639a626d88387efa9120fd, regionState=OPENING, regionLocation=d73269e94d72,38397,1732506072528 2024-11-25T03:41:15,788 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=4c0b379dd00b53bde73bca0e12f321ce, regionState=OPENING, regionLocation=d73269e94d72,38397,1732506072528 2024-11-25T03:41:15,789 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure df71ad9490ccea77dfed73e590672db8, server=d73269e94d72,38397,1732506072528}] 2024-11-25T03:41:15,789 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=706facb174cc9547b71f08aa672ddcee, ASSIGN because future has completed 2024-11-25T03:41:15,791 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=0698446036b7ae35179ebc9b07274db5, ASSIGN because future has completed 2024-11-25T03:41:15,794 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=2fc93e882c2b7c5acdb53e9cf1071dc8, ASSIGN because future has completed 2024-11-25T03:41:15,796 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2fc93e882c2b7c5acdb53e9cf1071dc8, server=d73269e94d72,38397,1732506072528}] 2024-11-25T03:41:15,796 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=0c260efb5b2a8a6137010d439efc8d47, ASSIGN because future has completed 2024-11-25T03:41:15,799 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=eb9ff3be6d639a626d88387efa9120fd, ASSIGN because future has completed 2024-11-25T03:41:15,801 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4c0b379dd00b53bde73bca0e12f321ce, ASSIGN because future has completed 2024-11-25T03:41:15,803 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4c0b379dd00b53bde73bca0e12f321ce, server=d73269e94d72,38397,1732506072528}] 2024-11-25T03:41:15,811 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb9ff3be6d639a626d88387efa9120fd, server=d73269e94d72,38397,1732506072528}] 2024-11-25T03:41:15,818 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0c260efb5b2a8a6137010d439efc8d47, server=d73269e94d72,38397,1732506072528}] 2024-11-25T03:41:15,826 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0698446036b7ae35179ebc9b07274db5, server=d73269e94d72,38397,1732506072528}] 2024-11-25T03:41:15,833 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 706facb174cc9547b71f08aa672ddcee, server=d73269e94d72,38397,1732506072528}] 2024-11-25T03:41:15,841 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=8a7d4164efdaed87f07b6f76a94819b3, regionState=OPENING, regionLocation=d73269e94d72,38397,1732506072528 2024-11-25T03:41:15,842 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=5e3a490d70b339ac73875ea7638f934f, regionState=OPENING, regionLocation=d73269e94d72,38397,1732506072528 2024-11-25T03:41:15,843 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=c769ca025a015554efcc98619008be96, regionState=OPENING, regionLocation=d73269e94d72,35981,1732506072374 2024-11-25T03:41:15,844 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=2b224c7c31e6ff664ce67ac884f573d7, regionState=OPENING, regionLocation=d73269e94d72,35981,1732506072374 2024-11-25T03:41:15,845 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=8b5ae4b02f22b24f40505bb660957004, regionState=OPENING, regionLocation=d73269e94d72,35981,1732506072374 2024-11-25T03:41:15,845 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8a7d4164efdaed87f07b6f76a94819b3, ASSIGN because future has completed 2024-11-25T03:41:15,845 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=922ebf14d68aba40da3a2e99777c777b, regionState=OPENING, regionLocation=d73269e94d72,35981,1732506072374 2024-11-25T03:41:15,846 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8a7d4164efdaed87f07b6f76a94819b3, server=d73269e94d72,38397,1732506072528}] 2024-11-25T03:41:15,848 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=5e3a490d70b339ac73875ea7638f934f, ASSIGN because future has completed 2024-11-25T03:41:15,849 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c769ca025a015554efcc98619008be96, ASSIGN because future has completed 2024-11-25T03:41:15,853 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=2b224c7c31e6ff664ce67ac884f573d7, ASSIGN because future has completed 2024-11-25T03:41:15,854 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8b5ae4b02f22b24f40505bb660957004, ASSIGN because future has completed 2024-11-25T03:41:15,856 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8b5ae4b02f22b24f40505bb660957004, server=d73269e94d72,35981,1732506072374}] 2024-11-25T03:41:15,856 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN because future has completed 2024-11-25T03:41:15,862 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 922ebf14d68aba40da3a2e99777c777b, server=d73269e94d72,35981,1732506072374}] 2024-11-25T03:41:15,868 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b224c7c31e6ff664ce67ac884f573d7, server=d73269e94d72,35981,1732506072374}] 2024-11-25T03:41:15,875 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure c769ca025a015554efcc98619008be96, server=d73269e94d72,35981,1732506072374}] 2024-11-25T03:41:15,882 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5e3a490d70b339ac73875ea7638f934f, server=d73269e94d72,38397,1732506072528}] 2024-11-25T03:41:15,888 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=4705fa4ae38907dd8b9783be2dff29da, regionState=OPENING, regionLocation=d73269e94d72,35981,1732506072374 2024-11-25T03:41:15,889 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=f5f64cb2042c701322ac00e312f65f16, regionState=OPENING, regionLocation=d73269e94d72,35981,1732506072374 2024-11-25T03:41:15,889 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=857a6b6999e909a9b73e698c69291205, regionState=OPENING, regionLocation=d73269e94d72,35981,1732506072374 2024-11-25T03:41:15,890 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=3b442507eaac3b5222a3c4f97ff80208, regionState=OPENING, regionLocation=d73269e94d72,35981,1732506072374 2024-11-25T03:41:15,891 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=5f812dacea03120e7730979dc37e38c6, regionState=OPENING, regionLocation=d73269e94d72,33145,1732506072477 2024-11-25T03:41:15,892 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=6aed1980a8b63b44c776d0990f923aa7, regionState=OPENING, regionLocation=d73269e94d72,33145,1732506072477 2024-11-25T03:41:15,892 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4705fa4ae38907dd8b9783be2dff29da, ASSIGN because future has completed 2024-11-25T03:41:15,892 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=745356a682ec14d9f6e2066d456cab06, regionState=OPENING, regionLocation=d73269e94d72,33145,1732506072477 2024-11-25T03:41:15,893 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4705fa4ae38907dd8b9783be2dff29da, server=d73269e94d72,35981,1732506072374}] 2024-11-25T03:41:15,895 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f5f64cb2042c701322ac00e312f65f16, ASSIGN because future has completed 2024-11-25T03:41:15,897 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN because future has completed 2024-11-25T03:41:15,898 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN because future has completed 2024-11-25T03:41:15,901 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=5f812dacea03120e7730979dc37e38c6, ASSIGN because future has completed 2024-11-25T03:41:15,903 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=6aed1980a8b63b44c776d0990f923aa7, ASSIGN because future has completed 2024-11-25T03:41:15,904 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=745356a682ec14d9f6e2066d456cab06, ASSIGN because future has completed 2024-11-25T03:41:15,904 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6aed1980a8b63b44c776d0990f923aa7, server=d73269e94d72,33145,1732506072477}] 2024-11-25T03:41:15,910 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 745356a682ec14d9f6e2066d456cab06, server=d73269e94d72,33145,1732506072477}] 2024-11-25T03:41:15,917 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5f812dacea03120e7730979dc37e38c6, server=d73269e94d72,33145,1732506072477}] 2024-11-25T03:41:15,922 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,35981,1732506072374}] 2024-11-25T03:41:15,927 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 857a6b6999e909a9b73e698c69291205, server=d73269e94d72,35981,1732506072374}] 2024-11-25T03:41:15,932 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5f64cb2042c701322ac00e312f65f16, server=d73269e94d72,35981,1732506072374}] 2024-11-25T03:41:15,936 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=5da71c9c32aa33cf6f067ea6d82ae84f, regionState=OPENING, regionLocation=d73269e94d72,33145,1732506072477 2024-11-25T03:41:15,936 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=a4b8682bf62ac734aa4174eba4ee1ddc, regionState=OPENING, regionLocation=d73269e94d72,33145,1732506072477 2024-11-25T03:41:15,937 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=5e791add66a4fee50be34aa83a6d502a, regionState=OPENING, regionLocation=d73269e94d72,33145,1732506072477 2024-11-25T03:41:15,938 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=bba1924bf10ad03aeb55628aeffc2363, regionState=OPENING, regionLocation=d73269e94d72,33145,1732506072477 2024-11-25T03:41:15,938 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=85a5fab26752f98e6966cd407e31db69, regionState=OPENING, regionLocation=d73269e94d72,33145,1732506072477 2024-11-25T03:41:15,939 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=c73ee0e3b504288792edda3ec9a26049, regionState=OPENING, regionLocation=d73269e94d72,33145,1732506072477 2024-11-25T03:41:15,940 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=5da71c9c32aa33cf6f067ea6d82ae84f, ASSIGN because future has completed 2024-11-25T03:41:15,940 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5da71c9c32aa33cf6f067ea6d82ae84f, server=d73269e94d72,33145,1732506072477}] 2024-11-25T03:41:15,943 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-25T03:41:15,944 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a4b8682bf62ac734aa4174eba4ee1ddc, ASSIGN because future has completed 2024-11-25T03:41:15,945 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure a4b8682bf62ac734aa4174eba4ee1ddc, server=d73269e94d72,33145,1732506072477}] 2024-11-25T03:41:15,945 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=5e791add66a4fee50be34aa83a6d502a, ASSIGN because future has completed 2024-11-25T03:41:15,946 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59525, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-25T03:41:15,947 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=bba1924bf10ad03aeb55628aeffc2363, ASSIGN because future has completed 2024-11-25T03:41:15,949 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=85a5fab26752f98e6966cd407e31db69, ASSIGN because future has completed 2024-11-25T03:41:15,950 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c73ee0e3b504288792edda3ec9a26049, ASSIGN because future has completed 2024-11-25T03:41:15,951 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure c73ee0e3b504288792edda3ec9a26049, server=d73269e94d72,33145,1732506072477}] 2024-11-25T03:41:15,952 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open test,ccc,1732506074951.5e3a490d70b339ac73875ea7638f934f. 2024-11-25T03:41:15,953 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => 5e3a490d70b339ac73875ea7638f934f, NAME => 'test,ccc,1732506074951.5e3a490d70b339ac73875ea7638f934f.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-25T03:41:15,953 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 5e3a490d70b339ac73875ea7638f934f 2024-11-25T03:41:15,953 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated test,ccc,1732506074951.5e3a490d70b339ac73875ea7638f934f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:41:15,954 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for 5e3a490d70b339ac73875ea7638f934f 2024-11-25T03:41:15,954 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for 5e3a490d70b339ac73875ea7638f934f 2024-11-25T03:41:15,956 INFO [StoreOpener-5e3a490d70b339ac73875ea7638f934f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 5e3a490d70b339ac73875ea7638f934f 2024-11-25T03:41:15,957 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 85a5fab26752f98e6966cd407e31db69, server=d73269e94d72,33145,1732506072477}] 2024-11-25T03:41:15,959 INFO [StoreOpener-5e3a490d70b339ac73875ea7638f934f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5e3a490d70b339ac73875ea7638f934f columnFamilyName family 2024-11-25T03:41:15,959 DEBUG [StoreOpener-5e3a490d70b339ac73875ea7638f934f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:41:15,960 INFO [StoreOpener-5e3a490d70b339ac73875ea7638f934f-1 {}] regionserver.HStore(327): Store=5e3a490d70b339ac73875ea7638f934f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:41:15,961 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for 5e3a490d70b339ac73875ea7638f934f 2024-11-25T03:41:15,962 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/5e3a490d70b339ac73875ea7638f934f 2024-11-25T03:41:15,963 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/5e3a490d70b339ac73875ea7638f934f 2024-11-25T03:41:15,963 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure bba1924bf10ad03aeb55628aeffc2363, server=d73269e94d72,33145,1732506072477}] 2024-11-25T03:41:15,964 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for 5e3a490d70b339ac73875ea7638f934f 2024-11-25T03:41:15,964 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for 5e3a490d70b339ac73875ea7638f934f 2024-11-25T03:41:15,967 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for 5e3a490d70b339ac73875ea7638f934f 2024-11-25T03:41:15,969 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5e791add66a4fee50be34aa83a6d502a, server=d73269e94d72,33145,1732506072477}] 2024-11-25T03:41:15,970 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/5e3a490d70b339ac73875ea7638f934f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:41:15,971 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened 5e3a490d70b339ac73875ea7638f934f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63210504, jitterRate=-0.05809009075164795}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:41:15,971 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5e3a490d70b339ac73875ea7638f934f 2024-11-25T03:41:15,972 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for 5e3a490d70b339ac73875ea7638f934f: Running coprocessor pre-open hook at 1732506075954Writing region info on filesystem at 1732506075954Initializing all the Stores at 1732506075955 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506075956 (+1 ms)Cleaning up temporary data from old regions at 1732506075964 (+8 ms)Running coprocessor post-open hooks at 1732506075971 (+7 ms)Region opened successfully at 1732506075972 (+1 ms) 2024-11-25T03:41:15,973 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1732506074951.5e3a490d70b339ac73875ea7638f934f., pid=43, masterSystemTime=1732506075943 2024-11-25T03:41:15,976 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ccc,1732506074951.5e3a490d70b339ac73875ea7638f934f. 2024-11-25T03:41:15,977 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened test,ccc,1732506074951.5e3a490d70b339ac73875ea7638f934f. 2024-11-25T03:41:15,977 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open test,iii,1732506074951.4c0b379dd00b53bde73bca0e12f321ce. 2024-11-25T03:41:15,977 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 4c0b379dd00b53bde73bca0e12f321ce, NAME => 'test,iii,1732506074951.4c0b379dd00b53bde73bca0e12f321ce.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-25T03:41:15,978 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4c0b379dd00b53bde73bca0e12f321ce 2024-11-25T03:41:15,978 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated test,iii,1732506074951.4c0b379dd00b53bde73bca0e12f321ce.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:41:15,978 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 4c0b379dd00b53bde73bca0e12f321ce 2024-11-25T03:41:15,978 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 4c0b379dd00b53bde73bca0e12f321ce 2024-11-25T03:41:15,980 INFO [StoreOpener-4c0b379dd00b53bde73bca0e12f321ce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4c0b379dd00b53bde73bca0e12f321ce 2024-11-25T03:41:15,981 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=5e3a490d70b339ac73875ea7638f934f, regionState=OPEN, openSeqNum=2, regionLocation=d73269e94d72,38397,1732506072528 2024-11-25T03:41:15,983 INFO [StoreOpener-4c0b379dd00b53bde73bca0e12f321ce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4c0b379dd00b53bde73bca0e12f321ce columnFamilyName family 2024-11-25T03:41:15,983 DEBUG [StoreOpener-4c0b379dd00b53bde73bca0e12f321ce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:41:15,985 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5e3a490d70b339ac73875ea7638f934f, server=d73269e94d72,38397,1732506072528 because future has completed 2024-11-25T03:41:15,988 INFO [StoreOpener-4c0b379dd00b53bde73bca0e12f321ce-1 {}] regionserver.HStore(327): Store=4c0b379dd00b53bde73bca0e12f321ce/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:41:15,988 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 4c0b379dd00b53bde73bca0e12f321ce 2024-11-25T03:41:15,990 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/4c0b379dd00b53bde73bca0e12f321ce 2024-11-25T03:41:15,990 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/4c0b379dd00b53bde73bca0e12f321ce 2024-11-25T03:41:15,991 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=7 2024-11-25T03:41:15,991 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 5e3a490d70b339ac73875ea7638f934f, server=d73269e94d72,38397,1732506072528 in 105 msec 2024-11-25T03:41:15,991 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 4c0b379dd00b53bde73bca0e12f321ce 2024-11-25T03:41:15,991 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 4c0b379dd00b53bde73bca0e12f321ce 2024-11-25T03:41:15,993 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=5e3a490d70b339ac73875ea7638f934f, ASSIGN in 374 msec 2024-11-25T03:41:15,994 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 4c0b379dd00b53bde73bca0e12f321ce 2024-11-25T03:41:15,997 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/4c0b379dd00b53bde73bca0e12f321ce/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:41:15,998 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 4c0b379dd00b53bde73bca0e12f321ce; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66186321, jitterRate=-0.013746961951255798}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:41:15,998 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4c0b379dd00b53bde73bca0e12f321ce 2024-11-25T03:41:15,998 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 4c0b379dd00b53bde73bca0e12f321ce: Running coprocessor pre-open hook at 1732506075978Writing region info on filesystem at 1732506075978Initializing all the Stores at 1732506075980 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506075980Cleaning up temporary data from old regions at 1732506075991 (+11 ms)Running coprocessor post-open hooks at 1732506075998 (+7 ms)Region opened successfully at 1732506075998 2024-11-25T03:41:15,999 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1732506074951.4c0b379dd00b53bde73bca0e12f321ce., pid=33, masterSystemTime=1732506075943 2024-11-25T03:41:16,002 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for test,iii,1732506074951.4c0b379dd00b53bde73bca0e12f321ce. 2024-11-25T03:41:16,002 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened test,iii,1732506074951.4c0b379dd00b53bde73bca0e12f321ce. 2024-11-25T03:41:16,002 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open test,lll,1732506074951.eb9ff3be6d639a626d88387efa9120fd. 2024-11-25T03:41:16,002 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => eb9ff3be6d639a626d88387efa9120fd, NAME => 'test,lll,1732506074951.eb9ff3be6d639a626d88387efa9120fd.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-25T03:41:16,003 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=4c0b379dd00b53bde73bca0e12f321ce, regionState=OPEN, openSeqNum=2, regionLocation=d73269e94d72,38397,1732506072528 2024-11-25T03:41:16,003 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test eb9ff3be6d639a626d88387efa9120fd 2024-11-25T03:41:16,003 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated test,lll,1732506074951.eb9ff3be6d639a626d88387efa9120fd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:41:16,003 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for eb9ff3be6d639a626d88387efa9120fd 2024-11-25T03:41:16,003 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for eb9ff3be6d639a626d88387efa9120fd 2024-11-25T03:41:16,005 INFO [StoreOpener-eb9ff3be6d639a626d88387efa9120fd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region eb9ff3be6d639a626d88387efa9120fd 2024-11-25T03:41:16,006 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4c0b379dd00b53bde73bca0e12f321ce, server=d73269e94d72,38397,1732506072528 because future has completed 2024-11-25T03:41:16,007 INFO [StoreOpener-eb9ff3be6d639a626d88387efa9120fd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eb9ff3be6d639a626d88387efa9120fd columnFamilyName family 2024-11-25T03:41:16,008 DEBUG [StoreOpener-eb9ff3be6d639a626d88387efa9120fd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:41:16,008 INFO [StoreOpener-eb9ff3be6d639a626d88387efa9120fd-1 {}] regionserver.HStore(327): Store=eb9ff3be6d639a626d88387efa9120fd/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:41:16,009 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for eb9ff3be6d639a626d88387efa9120fd 2024-11-25T03:41:16,009 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-25T03:41:16,010 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/eb9ff3be6d639a626d88387efa9120fd 2024-11-25T03:41:16,011 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/eb9ff3be6d639a626d88387efa9120fd 2024-11-25T03:41:16,011 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=13 2024-11-25T03:41:16,011 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56955, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-25T03:41:16,012 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure 4c0b379dd00b53bde73bca0e12f321ce, server=d73269e94d72,38397,1732506072528 in 204 msec 2024-11-25T03:41:16,012 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for eb9ff3be6d639a626d88387efa9120fd 2024-11-25T03:41:16,012 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for eb9ff3be6d639a626d88387efa9120fd 2024-11-25T03:41:16,015 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=4c0b379dd00b53bde73bca0e12f321ce, ASSIGN in 395 msec 2024-11-25T03:41:16,017 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for eb9ff3be6d639a626d88387efa9120fd 2024-11-25T03:41:16,026 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(132): Open test,nnn,1732506074951.4705fa4ae38907dd8b9783be2dff29da. 2024-11-25T03:41:16,026 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7752): Opening region: {ENCODED => 4705fa4ae38907dd8b9783be2dff29da, NAME => 'test,nnn,1732506074951.4705fa4ae38907dd8b9783be2dff29da.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-11-25T03:41:16,027 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4705fa4ae38907dd8b9783be2dff29da 2024-11-25T03:41:16,027 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(898): Instantiated test,nnn,1732506074951.4705fa4ae38907dd8b9783be2dff29da.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:41:16,027 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7794): checking encryption for 4705fa4ae38907dd8b9783be2dff29da 2024-11-25T03:41:16,028 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7797): checking classloading for 4705fa4ae38907dd8b9783be2dff29da 2024-11-25T03:41:16,028 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/eb9ff3be6d639a626d88387efa9120fd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:41:16,029 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened eb9ff3be6d639a626d88387efa9120fd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75339164, jitterRate=0.12264102697372437}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:41:16,029 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for eb9ff3be6d639a626d88387efa9120fd 2024-11-25T03:41:16,029 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for eb9ff3be6d639a626d88387efa9120fd: Running coprocessor pre-open hook at 1732506076003Writing region info on filesystem at 1732506076003Initializing all the Stores at 1732506076005 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506076005Cleaning up temporary data from old regions at 1732506076012 (+7 ms)Running coprocessor post-open hooks at 1732506076029 (+17 ms)Region opened successfully at 1732506076029 2024-11-25T03:41:16,031 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1732506074951.eb9ff3be6d639a626d88387efa9120fd., pid=34, masterSystemTime=1732506075943 2024-11-25T03:41:16,034 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1732506074951.eb9ff3be6d639a626d88387efa9120fd. 2024-11-25T03:41:16,034 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened test,lll,1732506074951.eb9ff3be6d639a626d88387efa9120fd. 2024-11-25T03:41:16,034 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open test,mmm,1732506074951.0c260efb5b2a8a6137010d439efc8d47. 2024-11-25T03:41:16,035 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => 0c260efb5b2a8a6137010d439efc8d47, NAME => 'test,mmm,1732506074951.0c260efb5b2a8a6137010d439efc8d47.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-25T03:41:16,035 INFO [StoreOpener-4705fa4ae38907dd8b9783be2dff29da-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4705fa4ae38907dd8b9783be2dff29da 2024-11-25T03:41:16,035 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=eb9ff3be6d639a626d88387efa9120fd, regionState=OPEN, openSeqNum=2, regionLocation=d73269e94d72,38397,1732506072528 2024-11-25T03:41:16,035 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 0c260efb5b2a8a6137010d439efc8d47 2024-11-25T03:41:16,035 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated test,mmm,1732506074951.0c260efb5b2a8a6137010d439efc8d47.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:41:16,035 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for 0c260efb5b2a8a6137010d439efc8d47 2024-11-25T03:41:16,035 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for 0c260efb5b2a8a6137010d439efc8d47 2024-11-25T03:41:16,039 INFO [StoreOpener-4705fa4ae38907dd8b9783be2dff29da-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4705fa4ae38907dd8b9783be2dff29da columnFamilyName family 2024-11-25T03:41:16,039 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb9ff3be6d639a626d88387efa9120fd, server=d73269e94d72,38397,1732506072528 because future has completed 2024-11-25T03:41:16,039 DEBUG [StoreOpener-4705fa4ae38907dd8b9783be2dff29da-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:41:16,041 INFO [StoreOpener-4705fa4ae38907dd8b9783be2dff29da-1 {}] regionserver.HStore(327): Store=4705fa4ae38907dd8b9783be2dff29da/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:41:16,041 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1038): replaying wal for 4705fa4ae38907dd8b9783be2dff29da 2024-11-25T03:41:16,043 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/4705fa4ae38907dd8b9783be2dff29da 2024-11-25T03:41:16,044 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/4705fa4ae38907dd8b9783be2dff29da 2024-11-25T03:41:16,045 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1048): stopping wal replay for 4705fa4ae38907dd8b9783be2dff29da 2024-11-25T03:41:16,045 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1060): Cleaning up temporary data for 4705fa4ae38907dd8b9783be2dff29da 2024-11-25T03:41:16,046 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=16 2024-11-25T03:41:16,047 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure eb9ff3be6d639a626d88387efa9120fd, server=d73269e94d72,38397,1732506072528 in 231 msec 2024-11-25T03:41:16,048 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1093): writing seq id for 4705fa4ae38907dd8b9783be2dff29da 2024-11-25T03:41:16,050 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=eb9ff3be6d639a626d88387efa9120fd, ASSIGN in 430 msec 2024-11-25T03:41:16,052 INFO [StoreOpener-0c260efb5b2a8a6137010d439efc8d47-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 0c260efb5b2a8a6137010d439efc8d47 2024-11-25T03:41:16,055 INFO [StoreOpener-0c260efb5b2a8a6137010d439efc8d47-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0c260efb5b2a8a6137010d439efc8d47 columnFamilyName family 2024-11-25T03:41:16,055 DEBUG [StoreOpener-0c260efb5b2a8a6137010d439efc8d47-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:41:16,056 INFO [StoreOpener-0c260efb5b2a8a6137010d439efc8d47-1 {}] regionserver.HStore(327): Store=0c260efb5b2a8a6137010d439efc8d47/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:41:16,056 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for 0c260efb5b2a8a6137010d439efc8d47 2024-11-25T03:41:16,057 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/0c260efb5b2a8a6137010d439efc8d47 2024-11-25T03:41:16,061 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/0c260efb5b2a8a6137010d439efc8d47 2024-11-25T03:41:16,061 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/4705fa4ae38907dd8b9783be2dff29da/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:41:16,062 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for 0c260efb5b2a8a6137010d439efc8d47 2024-11-25T03:41:16,062 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for 0c260efb5b2a8a6137010d439efc8d47 2024-11-25T03:41:16,062 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1114): Opened 4705fa4ae38907dd8b9783be2dff29da; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69366901, jitterRate=0.03364737331867218}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:41:16,062 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4705fa4ae38907dd8b9783be2dff29da 2024-11-25T03:41:16,062 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1006): Region open journal for 4705fa4ae38907dd8b9783be2dff29da: Running coprocessor pre-open hook at 1732506076031Writing region info on filesystem at 1732506076031Initializing all the Stores at 1732506076033 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506076033Cleaning up temporary data from old regions at 1732506076045 (+12 ms)Running coprocessor post-open hooks at 1732506076062 (+17 ms)Region opened successfully at 1732506076062 2024-11-25T03:41:16,065 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for 0c260efb5b2a8a6137010d439efc8d47 2024-11-25T03:41:16,065 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2236): Post open deploy tasks for test,nnn,1732506074951.4705fa4ae38907dd8b9783be2dff29da., pid=44, masterSystemTime=1732506076009 2024-11-25T03:41:16,068 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2266): Finished post open deploy task for test,nnn,1732506074951.4705fa4ae38907dd8b9783be2dff29da. 2024-11-25T03:41:16,069 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(153): Opened test,nnn,1732506074951.4705fa4ae38907dd8b9783be2dff29da. 2024-11-25T03:41:16,069 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(132): Open test,ggg,1732506074951.857a6b6999e909a9b73e698c69291205. 2024-11-25T03:41:16,069 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7752): Opening region: {ENCODED => 857a6b6999e909a9b73e698c69291205, NAME => 'test,ggg,1732506074951.857a6b6999e909a9b73e698c69291205.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-11-25T03:41:16,069 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=4705fa4ae38907dd8b9783be2dff29da, regionState=OPEN, openSeqNum=2, regionLocation=d73269e94d72,35981,1732506072374 2024-11-25T03:41:16,070 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 857a6b6999e909a9b73e698c69291205 2024-11-25T03:41:16,070 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(898): Instantiated test,ggg,1732506074951.857a6b6999e909a9b73e698c69291205.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:41:16,070 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7794): checking encryption for 857a6b6999e909a9b73e698c69291205 2024-11-25T03:41:16,070 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7797): checking classloading for 857a6b6999e909a9b73e698c69291205 2024-11-25T03:41:16,072 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4705fa4ae38907dd8b9783be2dff29da, server=d73269e94d72,35981,1732506072374 because future has completed 2024-11-25T03:41:16,073 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open test,xxx,1732506074951.5f812dacea03120e7730979dc37e38c6. 2024-11-25T03:41:16,073 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => 5f812dacea03120e7730979dc37e38c6, NAME => 'test,xxx,1732506074951.5f812dacea03120e7730979dc37e38c6.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-11-25T03:41:16,074 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 5f812dacea03120e7730979dc37e38c6 2024-11-25T03:41:16,074 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated test,xxx,1732506074951.5f812dacea03120e7730979dc37e38c6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:41:16,074 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for 5f812dacea03120e7730979dc37e38c6 2024-11-25T03:41:16,074 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for 5f812dacea03120e7730979dc37e38c6 2024-11-25T03:41:16,077 INFO [StoreOpener-857a6b6999e909a9b73e698c69291205-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 857a6b6999e909a9b73e698c69291205 2024-11-25T03:41:16,077 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/0c260efb5b2a8a6137010d439efc8d47/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:41:16,078 INFO [StoreOpener-5f812dacea03120e7730979dc37e38c6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 5f812dacea03120e7730979dc37e38c6 2024-11-25T03:41:16,078 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened 0c260efb5b2a8a6137010d439efc8d47; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64157197, jitterRate=-0.04398326575756073}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:41:16,078 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0c260efb5b2a8a6137010d439efc8d47 2024-11-25T03:41:16,078 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for 0c260efb5b2a8a6137010d439efc8d47: Running coprocessor pre-open hook at 1732506076035Writing region info on filesystem at 1732506076035Initializing all the Stores at 1732506076038 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506076038Cleaning up temporary data from old regions at 1732506076062 (+24 ms)Running coprocessor post-open hooks at 1732506076078 (+16 ms)Region opened successfully at 1732506076078 2024-11-25T03:41:16,079 INFO [StoreOpener-857a6b6999e909a9b73e698c69291205-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 857a6b6999e909a9b73e698c69291205 columnFamilyName family 2024-11-25T03:41:16,079 DEBUG [StoreOpener-857a6b6999e909a9b73e698c69291205-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:41:16,080 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1732506074951.0c260efb5b2a8a6137010d439efc8d47., pid=35, masterSystemTime=1732506075943 2024-11-25T03:41:16,080 INFO [StoreOpener-857a6b6999e909a9b73e698c69291205-1 {}] regionserver.HStore(327): Store=857a6b6999e909a9b73e698c69291205/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:41:16,081 INFO [StoreOpener-5f812dacea03120e7730979dc37e38c6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5f812dacea03120e7730979dc37e38c6 columnFamilyName family 2024-11-25T03:41:16,081 DEBUG [StoreOpener-5f812dacea03120e7730979dc37e38c6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:41:16,081 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1038): replaying wal for 857a6b6999e909a9b73e698c69291205 2024-11-25T03:41:16,082 INFO [StoreOpener-5f812dacea03120e7730979dc37e38c6-1 {}] regionserver.HStore(327): Store=5f812dacea03120e7730979dc37e38c6/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:41:16,082 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for 5f812dacea03120e7730979dc37e38c6 2024-11-25T03:41:16,083 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/857a6b6999e909a9b73e698c69291205 2024-11-25T03:41:16,082 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=18 2024-11-25T03:41:16,083 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure 4705fa4ae38907dd8b9783be2dff29da, server=d73269e94d72,35981,1732506072374 in 181 msec 2024-11-25T03:41:16,084 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1732506074951.0c260efb5b2a8a6137010d439efc8d47. 2024-11-25T03:41:16,084 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened test,mmm,1732506074951.0c260efb5b2a8a6137010d439efc8d47. 2024-11-25T03:41:16,084 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/5f812dacea03120e7730979dc37e38c6 2024-11-25T03:41:16,084 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open test,eee,1732506074951.8a7d4164efdaed87f07b6f76a94819b3. 2024-11-25T03:41:16,084 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => 8a7d4164efdaed87f07b6f76a94819b3, NAME => 'test,eee,1732506074951.8a7d4164efdaed87f07b6f76a94819b3.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-25T03:41:16,084 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/857a6b6999e909a9b73e698c69291205 2024-11-25T03:41:16,085 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/5f812dacea03120e7730979dc37e38c6 2024-11-25T03:41:16,085 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8a7d4164efdaed87f07b6f76a94819b3 2024-11-25T03:41:16,085 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated test,eee,1732506074951.8a7d4164efdaed87f07b6f76a94819b3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:41:16,085 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for 8a7d4164efdaed87f07b6f76a94819b3 2024-11-25T03:41:16,085 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for 8a7d4164efdaed87f07b6f76a94819b3 2024-11-25T03:41:16,085 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1048): stopping wal replay for 857a6b6999e909a9b73e698c69291205 2024-11-25T03:41:16,085 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1060): Cleaning up temporary data for 857a6b6999e909a9b73e698c69291205 2024-11-25T03:41:16,085 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for 5f812dacea03120e7730979dc37e38c6 2024-11-25T03:41:16,086 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for 5f812dacea03120e7730979dc37e38c6 2024-11-25T03:41:16,087 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=4705fa4ae38907dd8b9783be2dff29da, ASSIGN in 466 msec 2024-11-25T03:41:16,087 INFO [StoreOpener-8a7d4164efdaed87f07b6f76a94819b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8a7d4164efdaed87f07b6f76a94819b3 2024-11-25T03:41:16,089 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): writing seq id for 857a6b6999e909a9b73e698c69291205 2024-11-25T03:41:16,090 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=0c260efb5b2a8a6137010d439efc8d47, regionState=OPEN, openSeqNum=2, regionLocation=d73269e94d72,38397,1732506072528 2024-11-25T03:41:16,091 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for 5f812dacea03120e7730979dc37e38c6 2024-11-25T03:41:16,091 INFO [StoreOpener-8a7d4164efdaed87f07b6f76a94819b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a7d4164efdaed87f07b6f76a94819b3 columnFamilyName family 2024-11-25T03:41:16,092 DEBUG [StoreOpener-8a7d4164efdaed87f07b6f76a94819b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:41:16,093 INFO [StoreOpener-8a7d4164efdaed87f07b6f76a94819b3-1 {}] regionserver.HStore(327): Store=8a7d4164efdaed87f07b6f76a94819b3/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:41:16,093 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0c260efb5b2a8a6137010d439efc8d47, server=d73269e94d72,38397,1732506072528 because future has completed 2024-11-25T03:41:16,093 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for 8a7d4164efdaed87f07b6f76a94819b3 2024-11-25T03:41:16,094 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/857a6b6999e909a9b73e698c69291205/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:41:16,095 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/8a7d4164efdaed87f07b6f76a94819b3 2024-11-25T03:41:16,096 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1114): Opened 857a6b6999e909a9b73e698c69291205; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62758385, jitterRate=-0.06482718884944916}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:41:16,096 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 857a6b6999e909a9b73e698c69291205 2024-11-25T03:41:16,096 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1006): Region open journal for 857a6b6999e909a9b73e698c69291205: Running coprocessor pre-open hook at 1732506076070Writing region info on filesystem at 1732506076070Initializing all the Stores at 1732506076073 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506076073Cleaning up temporary data from old regions at 1732506076085 (+12 ms)Running coprocessor post-open hooks at 1732506076096 (+11 ms)Region opened successfully at 1732506076096 2024-11-25T03:41:16,096 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/8a7d4164efdaed87f07b6f76a94819b3 2024-11-25T03:41:16,097 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for 8a7d4164efdaed87f07b6f76a94819b3 2024-11-25T03:41:16,097 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for 8a7d4164efdaed87f07b6f76a94819b3 2024-11-25T03:41:16,097 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ggg,1732506074951.857a6b6999e909a9b73e698c69291205., pid=49, masterSystemTime=1732506076009 2024-11-25T03:41:16,097 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/5f812dacea03120e7730979dc37e38c6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:41:16,100 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened 5f812dacea03120e7730979dc37e38c6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67331845, jitterRate=0.0033226758241653442}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:41:16,100 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5f812dacea03120e7730979dc37e38c6 2024-11-25T03:41:16,100 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for 5f812dacea03120e7730979dc37e38c6: Running coprocessor pre-open hook at 1732506076074Writing region info on filesystem at 1732506076074Initializing all the Stores at 1732506076076 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506076076Cleaning up temporary data from old regions at 1732506076086 (+10 ms)Running coprocessor post-open hooks at 1732506076100 (+14 ms)Region opened successfully at 1732506076100 2024-11-25T03:41:16,101 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for 8a7d4164efdaed87f07b6f76a94819b3 2024-11-25T03:41:16,101 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ggg,1732506074951.857a6b6999e909a9b73e698c69291205. 2024-11-25T03:41:16,101 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(153): Opened test,ggg,1732506074951.857a6b6999e909a9b73e698c69291205. 2024-11-25T03:41:16,101 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open test,ttt,1732506074951.8b5ae4b02f22b24f40505bb660957004. 2024-11-25T03:41:16,101 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1732506074951.5f812dacea03120e7730979dc37e38c6., pid=47, masterSystemTime=1732506076057 2024-11-25T03:41:16,102 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => 8b5ae4b02f22b24f40505bb660957004, NAME => 'test,ttt,1732506074951.8b5ae4b02f22b24f40505bb660957004.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-25T03:41:16,102 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8b5ae4b02f22b24f40505bb660957004 2024-11-25T03:41:16,102 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated test,ttt,1732506074951.8b5ae4b02f22b24f40505bb660957004.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:41:16,102 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for 8b5ae4b02f22b24f40505bb660957004 2024-11-25T03:41:16,102 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for 8b5ae4b02f22b24f40505bb660957004 2024-11-25T03:41:16,105 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=17 2024-11-25T03:41:16,105 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure 0c260efb5b2a8a6137010d439efc8d47, server=d73269e94d72,38397,1732506072528 in 279 msec 2024-11-25T03:41:16,105 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1732506074951.5f812dacea03120e7730979dc37e38c6. 2024-11-25T03:41:16,105 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened test,xxx,1732506074951.5f812dacea03120e7730979dc37e38c6. 2024-11-25T03:41:16,106 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open test,fff,1732506074951.bba1924bf10ad03aeb55628aeffc2363. 2024-11-25T03:41:16,106 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => bba1924bf10ad03aeb55628aeffc2363, NAME => 'test,fff,1732506074951.bba1924bf10ad03aeb55628aeffc2363.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-25T03:41:16,106 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/8a7d4164efdaed87f07b6f76a94819b3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:41:16,106 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test bba1924bf10ad03aeb55628aeffc2363 2024-11-25T03:41:16,106 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated test,fff,1732506074951.bba1924bf10ad03aeb55628aeffc2363.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:41:16,106 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for bba1924bf10ad03aeb55628aeffc2363 2024-11-25T03:41:16,107 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for bba1924bf10ad03aeb55628aeffc2363 2024-11-25T03:41:16,107 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened 8a7d4164efdaed87f07b6f76a94819b3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67864303, jitterRate=0.01125691831111908}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:41:16,107 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8a7d4164efdaed87f07b6f76a94819b3 2024-11-25T03:41:16,107 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for 8a7d4164efdaed87f07b6f76a94819b3: Running coprocessor pre-open hook at 1732506076085Writing region info on filesystem at 1732506076085Initializing all the Stores at 1732506076087 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506076087Cleaning up temporary data from old regions at 1732506076097 (+10 ms)Running coprocessor post-open hooks at 1732506076107 (+10 ms)Region opened successfully at 1732506076107 2024-11-25T03:41:16,107 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=5f812dacea03120e7730979dc37e38c6, regionState=OPEN, openSeqNum=2, regionLocation=d73269e94d72,33145,1732506072477 2024-11-25T03:41:16,109 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1732506074951.8a7d4164efdaed87f07b6f76a94819b3., pid=38, masterSystemTime=1732506075943 2024-11-25T03:41:16,111 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1732506074951.8a7d4164efdaed87f07b6f76a94819b3. 2024-11-25T03:41:16,112 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened test,eee,1732506074951.8a7d4164efdaed87f07b6f76a94819b3. 2024-11-25T03:41:16,112 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5f812dacea03120e7730979dc37e38c6, server=d73269e94d72,33145,1732506072477 because future has completed 2024-11-25T03:41:16,112 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open test,sss,1732506074951.0698446036b7ae35179ebc9b07274db5. 2024-11-25T03:41:16,112 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => 0698446036b7ae35179ebc9b07274db5, NAME => 'test,sss,1732506074951.0698446036b7ae35179ebc9b07274db5.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-25T03:41:16,112 INFO [StoreOpener-8b5ae4b02f22b24f40505bb660957004-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8b5ae4b02f22b24f40505bb660957004 2024-11-25T03:41:16,113 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 0698446036b7ae35179ebc9b07274db5 2024-11-25T03:41:16,113 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated test,sss,1732506074951.0698446036b7ae35179ebc9b07274db5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:41:16,113 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for 0698446036b7ae35179ebc9b07274db5 2024-11-25T03:41:16,113 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for 0698446036b7ae35179ebc9b07274db5 2024-11-25T03:41:16,115 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=0c260efb5b2a8a6137010d439efc8d47, ASSIGN in 493 msec 2024-11-25T03:41:16,116 INFO [StoreOpener-8b5ae4b02f22b24f40505bb660957004-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8b5ae4b02f22b24f40505bb660957004 columnFamilyName family 2024-11-25T03:41:16,117 DEBUG [StoreOpener-8b5ae4b02f22b24f40505bb660957004-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:41:16,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T03:41:16,117 INFO [StoreOpener-bba1924bf10ad03aeb55628aeffc2363-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region bba1924bf10ad03aeb55628aeffc2363 2024-11-25T03:41:16,118 INFO [StoreOpener-0698446036b7ae35179ebc9b07274db5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 0698446036b7ae35179ebc9b07274db5 2024-11-25T03:41:16,118 INFO [StoreOpener-8b5ae4b02f22b24f40505bb660957004-1 {}] regionserver.HStore(327): Store=8b5ae4b02f22b24f40505bb660957004/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:41:16,118 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for 8b5ae4b02f22b24f40505bb660957004 2024-11-25T03:41:16,120 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/8b5ae4b02f22b24f40505bb660957004 2024-11-25T03:41:16,120 INFO [StoreOpener-bba1924bf10ad03aeb55628aeffc2363-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bba1924bf10ad03aeb55628aeffc2363 columnFamilyName family 2024-11-25T03:41:16,120 DEBUG [StoreOpener-bba1924bf10ad03aeb55628aeffc2363-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:41:16,120 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/8b5ae4b02f22b24f40505bb660957004 2024-11-25T03:41:16,121 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for 8b5ae4b02f22b24f40505bb660957004 2024-11-25T03:41:16,121 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for 8b5ae4b02f22b24f40505bb660957004 2024-11-25T03:41:16,121 INFO [StoreOpener-bba1924bf10ad03aeb55628aeffc2363-1 {}] regionserver.HStore(327): Store=bba1924bf10ad03aeb55628aeffc2363/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:41:16,121 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for bba1924bf10ad03aeb55628aeffc2363 2024-11-25T03:41:16,121 INFO [StoreOpener-0698446036b7ae35179ebc9b07274db5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0698446036b7ae35179ebc9b07274db5 columnFamilyName family 2024-11-25T03:41:16,121 DEBUG [StoreOpener-0698446036b7ae35179ebc9b07274db5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:41:16,122 INFO [StoreOpener-0698446036b7ae35179ebc9b07274db5-1 {}] regionserver.HStore(327): Store=0698446036b7ae35179ebc9b07274db5/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:41:16,122 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for 0698446036b7ae35179ebc9b07274db5 2024-11-25T03:41:16,123 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/bba1924bf10ad03aeb55628aeffc2363 2024-11-25T03:41:16,123 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/bba1924bf10ad03aeb55628aeffc2363 2024-11-25T03:41:16,123 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=28 2024-11-25T03:41:16,124 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/0698446036b7ae35179ebc9b07274db5 2024-11-25T03:41:16,124 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=28, state=SUCCESS, hasLock=false; OpenRegionProcedure 5f812dacea03120e7730979dc37e38c6, server=d73269e94d72,33145,1732506072477 in 201 msec 2024-11-25T03:41:16,124 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for bba1924bf10ad03aeb55628aeffc2363 2024-11-25T03:41:16,124 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/0698446036b7ae35179ebc9b07274db5 2024-11-25T03:41:16,124 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for bba1924bf10ad03aeb55628aeffc2363 2024-11-25T03:41:16,124 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for 8b5ae4b02f22b24f40505bb660957004 2024-11-25T03:41:16,125 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for 0698446036b7ae35179ebc9b07274db5 2024-11-25T03:41:16,125 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for 0698446036b7ae35179ebc9b07274db5 2024-11-25T03:41:16,127 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for bba1924bf10ad03aeb55628aeffc2363 2024-11-25T03:41:16,128 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for 0698446036b7ae35179ebc9b07274db5 2024-11-25T03:41:16,128 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/8b5ae4b02f22b24f40505bb660957004/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:41:16,129 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened 8b5ae4b02f22b24f40505bb660957004; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60309255, jitterRate=-0.10132206976413727}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:41:16,129 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=5f812dacea03120e7730979dc37e38c6, ASSIGN in 507 msec 2024-11-25T03:41:16,129 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8b5ae4b02f22b24f40505bb660957004 2024-11-25T03:41:16,129 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for 8b5ae4b02f22b24f40505bb660957004: Running coprocessor pre-open hook at 1732506076103Writing region info on filesystem at 1732506076103Initializing all the Stores at 1732506076106 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506076107 (+1 ms)Cleaning up temporary data from old regions at 1732506076121 (+14 ms)Running coprocessor post-open hooks at 1732506076129 (+8 ms)Region opened successfully at 1732506076129 2024-11-25T03:41:16,130 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1732506074951.8b5ae4b02f22b24f40505bb660957004., pid=39, masterSystemTime=1732506076009 2024-11-25T03:41:16,131 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/bba1924bf10ad03aeb55628aeffc2363/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:41:16,132 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened bba1924bf10ad03aeb55628aeffc2363; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75163484, jitterRate=0.12002319097518921}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:41:16,132 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bba1924bf10ad03aeb55628aeffc2363 2024-11-25T03:41:16,133 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for bba1924bf10ad03aeb55628aeffc2363: Running coprocessor pre-open hook at 1732506076107Writing region info on filesystem at 1732506076107Initializing all the Stores at 1732506076109 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506076109Cleaning up temporary data from old regions at 1732506076124 (+15 ms)Running coprocessor post-open hooks at 1732506076132 (+8 ms)Region opened successfully at 1732506076133 (+1 ms) 2024-11-25T03:41:16,133 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/0698446036b7ae35179ebc9b07274db5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:41:16,133 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=8a7d4164efdaed87f07b6f76a94819b3, regionState=OPEN, openSeqNum=2, regionLocation=d73269e94d72,38397,1732506072528 2024-11-25T03:41:16,134 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened 0698446036b7ae35179ebc9b07274db5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73681650, jitterRate=0.09794214367866516}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:41:16,134 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0698446036b7ae35179ebc9b07274db5 2024-11-25T03:41:16,134 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for 0698446036b7ae35179ebc9b07274db5: Running coprocessor pre-open hook at 1732506076113Writing region info on filesystem at 1732506076113Initializing all the Stores at 1732506076115 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506076115Cleaning up temporary data from old regions at 1732506076125 (+10 ms)Running coprocessor post-open hooks at 1732506076134 (+9 ms)Region opened successfully at 1732506076134 2024-11-25T03:41:16,135 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1732506074951.8b5ae4b02f22b24f40505bb660957004. 2024-11-25T03:41:16,135 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened test,ttt,1732506074951.8b5ae4b02f22b24f40505bb660957004. 2024-11-25T03:41:16,135 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(132): Open test,zzz,1732506074951.c769ca025a015554efcc98619008be96. 2024-11-25T03:41:16,135 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7752): Opening region: {ENCODED => c769ca025a015554efcc98619008be96, NAME => 'test,zzz,1732506074951.c769ca025a015554efcc98619008be96.', STARTKEY => 'zzz', ENDKEY => ''} 2024-11-25T03:41:16,135 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1732506074951.bba1924bf10ad03aeb55628aeffc2363., pid=55, masterSystemTime=1732506076057 2024-11-25T03:41:16,136 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c769ca025a015554efcc98619008be96 2024-11-25T03:41:16,136 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(898): Instantiated test,zzz,1732506074951.c769ca025a015554efcc98619008be96.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:41:16,136 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7794): checking encryption for c769ca025a015554efcc98619008be96 2024-11-25T03:41:16,136 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7797): checking classloading for c769ca025a015554efcc98619008be96 2024-11-25T03:41:16,138 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1732506074951.0698446036b7ae35179ebc9b07274db5., pid=36, masterSystemTime=1732506075943 2024-11-25T03:41:16,138 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8a7d4164efdaed87f07b6f76a94819b3, server=d73269e94d72,38397,1732506072528 because future has completed 2024-11-25T03:41:16,139 INFO [StoreOpener-c769ca025a015554efcc98619008be96-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c769ca025a015554efcc98619008be96 2024-11-25T03:41:16,140 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1732506074951.bba1924bf10ad03aeb55628aeffc2363. 2024-11-25T03:41:16,140 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened test,fff,1732506074951.bba1924bf10ad03aeb55628aeffc2363. 2024-11-25T03:41:16,140 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=8b5ae4b02f22b24f40505bb660957004, regionState=OPEN, openSeqNum=2, regionLocation=d73269e94d72,35981,1732506072374 2024-11-25T03:41:16,140 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(132): Open test,ddd,1732506074951.85a5fab26752f98e6966cd407e31db69. 2024-11-25T03:41:16,141 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7752): Opening region: {ENCODED => 85a5fab26752f98e6966cd407e31db69, NAME => 'test,ddd,1732506074951.85a5fab26752f98e6966cd407e31db69.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-11-25T03:41:16,141 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 85a5fab26752f98e6966cd407e31db69 2024-11-25T03:41:16,141 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(898): Instantiated test,ddd,1732506074951.85a5fab26752f98e6966cd407e31db69.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:41:16,141 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7794): checking encryption for 85a5fab26752f98e6966cd407e31db69 2024-11-25T03:41:16,141 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7797): checking classloading for 85a5fab26752f98e6966cd407e31db69 2024-11-25T03:41:16,142 INFO [StoreOpener-c769ca025a015554efcc98619008be96-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c769ca025a015554efcc98619008be96 columnFamilyName family 2024-11-25T03:41:16,142 DEBUG [StoreOpener-c769ca025a015554efcc98619008be96-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:41:16,142 INFO [StoreOpener-c769ca025a015554efcc98619008be96-1 {}] regionserver.HStore(327): Store=c769ca025a015554efcc98619008be96/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:41:16,143 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1038): replaying wal for c769ca025a015554efcc98619008be96 2024-11-25T03:41:16,144 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1732506074951.0698446036b7ae35179ebc9b07274db5. 2024-11-25T03:41:16,144 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened test,sss,1732506074951.0698446036b7ae35179ebc9b07274db5. 2024-11-25T03:41:16,144 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open test,yyy,1732506074951.df71ad9490ccea77dfed73e590672db8. 2024-11-25T03:41:16,144 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => df71ad9490ccea77dfed73e590672db8, NAME => 'test,yyy,1732506074951.df71ad9490ccea77dfed73e590672db8.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-25T03:41:16,145 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test df71ad9490ccea77dfed73e590672db8 2024-11-25T03:41:16,145 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated test,yyy,1732506074951.df71ad9490ccea77dfed73e590672db8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:41:16,145 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for df71ad9490ccea77dfed73e590672db8 2024-11-25T03:41:16,145 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for df71ad9490ccea77dfed73e590672db8 2024-11-25T03:41:16,146 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/c769ca025a015554efcc98619008be96 2024-11-25T03:41:16,146 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8b5ae4b02f22b24f40505bb660957004, server=d73269e94d72,35981,1732506072374 because future has completed 2024-11-25T03:41:16,146 INFO [StoreOpener-85a5fab26752f98e6966cd407e31db69-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 85a5fab26752f98e6966cd407e31db69 2024-11-25T03:41:16,146 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/c769ca025a015554efcc98619008be96 2024-11-25T03:41:16,147 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=0698446036b7ae35179ebc9b07274db5, regionState=OPEN, openSeqNum=2, regionLocation=d73269e94d72,38397,1732506072528 2024-11-25T03:41:16,147 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1048): stopping wal replay for c769ca025a015554efcc98619008be96 2024-11-25T03:41:16,147 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1060): Cleaning up temporary data for c769ca025a015554efcc98619008be96 2024-11-25T03:41:16,148 INFO [StoreOpener-df71ad9490ccea77dfed73e590672db8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region df71ad9490ccea77dfed73e590672db8 2024-11-25T03:41:16,150 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1093): writing seq id for c769ca025a015554efcc98619008be96 2024-11-25T03:41:16,151 INFO [StoreOpener-85a5fab26752f98e6966cd407e31db69-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 85a5fab26752f98e6966cd407e31db69 columnFamilyName family 2024-11-25T03:41:16,151 DEBUG [StoreOpener-85a5fab26752f98e6966cd407e31db69-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:41:16,151 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0698446036b7ae35179ebc9b07274db5, server=d73269e94d72,38397,1732506072528 because future has completed 2024-11-25T03:41:16,152 INFO [StoreOpener-df71ad9490ccea77dfed73e590672db8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region df71ad9490ccea77dfed73e590672db8 columnFamilyName family 2024-11-25T03:41:16,152 DEBUG [StoreOpener-df71ad9490ccea77dfed73e590672db8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:41:16,152 INFO [StoreOpener-85a5fab26752f98e6966cd407e31db69-1 {}] regionserver.HStore(327): Store=85a5fab26752f98e6966cd407e31db69/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:41:16,153 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1038): replaying wal for 85a5fab26752f98e6966cd407e31db69 2024-11-25T03:41:16,153 INFO [StoreOpener-df71ad9490ccea77dfed73e590672db8-1 {}] regionserver.HStore(327): Store=df71ad9490ccea77dfed73e590672db8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:41:16,153 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for df71ad9490ccea77dfed73e590672db8 2024-11-25T03:41:16,154 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/85a5fab26752f98e6966cd407e31db69 2024-11-25T03:41:16,154 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/85a5fab26752f98e6966cd407e31db69 2024-11-25T03:41:16,155 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1048): stopping wal replay for 85a5fab26752f98e6966cd407e31db69 2024-11-25T03:41:16,155 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1060): Cleaning up temporary data for 85a5fab26752f98e6966cd407e31db69 2024-11-25T03:41:16,155 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/df71ad9490ccea77dfed73e590672db8 2024-11-25T03:41:16,156 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/df71ad9490ccea77dfed73e590672db8 2024-11-25T03:41:16,157 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for df71ad9490ccea77dfed73e590672db8 2024-11-25T03:41:16,157 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=24 2024-11-25T03:41:16,157 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for df71ad9490ccea77dfed73e590672db8 2024-11-25T03:41:16,157 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure 8b5ae4b02f22b24f40505bb660957004, server=d73269e94d72,35981,1732506072374 in 295 msec 2024-11-25T03:41:16,158 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1093): writing seq id for 85a5fab26752f98e6966cd407e31db69 2024-11-25T03:41:16,160 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/c769ca025a015554efcc98619008be96/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:41:16,160 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for df71ad9490ccea77dfed73e590672db8 2024-11-25T03:41:16,161 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8b5ae4b02f22b24f40505bb660957004, ASSIGN in 540 msec 2024-11-25T03:41:16,161 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1114): Opened c769ca025a015554efcc98619008be96; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67653568, jitterRate=0.008116722106933594}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:41:16,161 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c769ca025a015554efcc98619008be96 2024-11-25T03:41:16,161 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1006): Region open journal for c769ca025a015554efcc98619008be96: Running coprocessor pre-open hook at 1732506076136Writing region info on filesystem at 1732506076136Initializing all the Stores at 1732506076139 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506076139Cleaning up temporary data from old regions at 1732506076147 (+8 ms)Running coprocessor post-open hooks at 1732506076161 (+14 ms)Region opened successfully at 1732506076161 2024-11-25T03:41:16,162 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2236): Post open deploy tasks for test,zzz,1732506074951.c769ca025a015554efcc98619008be96., pid=42, masterSystemTime=1732506076009 2024-11-25T03:41:16,163 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/85a5fab26752f98e6966cd407e31db69/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:41:16,164 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1114): Opened 85a5fab26752f98e6966cd407e31db69; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73380030, jitterRate=0.09344765543937683}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:41:16,164 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 85a5fab26752f98e6966cd407e31db69 2024-11-25T03:41:16,165 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1006): Region open journal for 85a5fab26752f98e6966cd407e31db69: Running coprocessor pre-open hook at 1732506076141Writing region info on filesystem at 1732506076141Initializing all the Stores at 1732506076143 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506076143Cleaning up temporary data from old regions at 1732506076155 (+12 ms)Running coprocessor post-open hooks at 1732506076164 (+9 ms)Region opened successfully at 1732506076164 2024-11-25T03:41:16,166 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ddd,1732506074951.85a5fab26752f98e6966cd407e31db69., pid=54, masterSystemTime=1732506076057 2024-11-25T03:41:16,167 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2266): Finished post open deploy task for test,zzz,1732506074951.c769ca025a015554efcc98619008be96. 2024-11-25T03:41:16,167 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(153): Opened test,zzz,1732506074951.c769ca025a015554efcc98619008be96. 2024-11-25T03:41:16,167 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/df71ad9490ccea77dfed73e590672db8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:41:16,167 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(132): Open test,,1732506074951.3b442507eaac3b5222a3c4f97ff80208. 2024-11-25T03:41:16,167 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7752): Opening region: {ENCODED => 3b442507eaac3b5222a3c4f97ff80208, NAME => 'test,,1732506074951.3b442507eaac3b5222a3c4f97ff80208.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-25T03:41:16,168 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened df71ad9490ccea77dfed73e590672db8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63815354, jitterRate=-0.049077123403549194}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:41:16,168 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 3b442507eaac3b5222a3c4f97ff80208 2024-11-25T03:41:16,168 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for df71ad9490ccea77dfed73e590672db8 2024-11-25T03:41:16,168 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(898): Instantiated test,,1732506074951.3b442507eaac3b5222a3c4f97ff80208.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:41:16,168 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for df71ad9490ccea77dfed73e590672db8: Running coprocessor pre-open hook at 1732506076145Writing region info on filesystem at 1732506076145Initializing all the Stores at 1732506076147 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506076147Cleaning up temporary data from old regions at 1732506076157 (+10 ms)Running coprocessor post-open hooks at 1732506076168 (+11 ms)Region opened successfully at 1732506076168 2024-11-25T03:41:16,168 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7794): checking encryption for 3b442507eaac3b5222a3c4f97ff80208 2024-11-25T03:41:16,168 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7797): checking classloading for 3b442507eaac3b5222a3c4f97ff80208 2024-11-25T03:41:16,168 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=23 2024-11-25T03:41:16,169 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 0698446036b7ae35179ebc9b07274db5, server=d73269e94d72,38397,1732506072528 in 336 msec 2024-11-25T03:41:16,170 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1732506074951.df71ad9490ccea77dfed73e590672db8., pid=31, masterSystemTime=1732506075943 2024-11-25T03:41:16,170 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ddd,1732506074951.85a5fab26752f98e6966cd407e31db69. 2024-11-25T03:41:16,170 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(153): Opened test,ddd,1732506074951.85a5fab26752f98e6966cd407e31db69. 2024-11-25T03:41:16,171 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open test,uuu,1732506074951.6aed1980a8b63b44c776d0990f923aa7. 2024-11-25T03:41:16,171 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => 6aed1980a8b63b44c776d0990f923aa7, NAME => 'test,uuu,1732506074951.6aed1980a8b63b44c776d0990f923aa7.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-25T03:41:16,171 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 6aed1980a8b63b44c776d0990f923aa7 2024-11-25T03:41:16,171 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated test,uuu,1732506074951.6aed1980a8b63b44c776d0990f923aa7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:41:16,172 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for 6aed1980a8b63b44c776d0990f923aa7 2024-11-25T03:41:16,172 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for 6aed1980a8b63b44c776d0990f923aa7 2024-11-25T03:41:16,172 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=0698446036b7ae35179ebc9b07274db5, ASSIGN in 552 msec 2024-11-25T03:41:16,172 INFO [StoreOpener-3b442507eaac3b5222a3c4f97ff80208-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 3b442507eaac3b5222a3c4f97ff80208 2024-11-25T03:41:16,173 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1732506074951.df71ad9490ccea77dfed73e590672db8. 2024-11-25T03:41:16,173 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened test,yyy,1732506074951.df71ad9490ccea77dfed73e590672db8. 2024-11-25T03:41:16,174 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open test,vvv,1732506074951.706facb174cc9547b71f08aa672ddcee. 2024-11-25T03:41:16,174 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 706facb174cc9547b71f08aa672ddcee, NAME => 'test,vvv,1732506074951.706facb174cc9547b71f08aa672ddcee.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-25T03:41:16,174 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 706facb174cc9547b71f08aa672ddcee 2024-11-25T03:41:16,174 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated test,vvv,1732506074951.706facb174cc9547b71f08aa672ddcee.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:41:16,174 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 706facb174cc9547b71f08aa672ddcee 2024-11-25T03:41:16,174 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 706facb174cc9547b71f08aa672ddcee 2024-11-25T03:41:16,174 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=df71ad9490ccea77dfed73e590672db8, regionState=OPEN, openSeqNum=2, regionLocation=d73269e94d72,38397,1732506072528 2024-11-25T03:41:16,176 INFO [StoreOpener-6aed1980a8b63b44c776d0990f923aa7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 6aed1980a8b63b44c776d0990f923aa7 2024-11-25T03:41:16,176 INFO [StoreOpener-3b442507eaac3b5222a3c4f97ff80208-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3b442507eaac3b5222a3c4f97ff80208 columnFamilyName family 2024-11-25T03:41:16,176 DEBUG [StoreOpener-3b442507eaac3b5222a3c4f97ff80208-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:41:16,178 INFO [StoreOpener-3b442507eaac3b5222a3c4f97ff80208-1 {}] regionserver.HStore(327): Store=3b442507eaac3b5222a3c4f97ff80208/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:41:16,179 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1038): replaying wal for 3b442507eaac3b5222a3c4f97ff80208 2024-11-25T03:41:16,179 INFO [StoreOpener-6aed1980a8b63b44c776d0990f923aa7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6aed1980a8b63b44c776d0990f923aa7 columnFamilyName family 2024-11-25T03:41:16,179 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure df71ad9490ccea77dfed73e590672db8, server=d73269e94d72,38397,1732506072528 because future has completed 2024-11-25T03:41:16,179 DEBUG [StoreOpener-6aed1980a8b63b44c776d0990f923aa7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:41:16,180 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/3b442507eaac3b5222a3c4f97ff80208 2024-11-25T03:41:16,180 INFO [StoreOpener-6aed1980a8b63b44c776d0990f923aa7-1 {}] regionserver.HStore(327): Store=6aed1980a8b63b44c776d0990f923aa7/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:41:16,180 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=85a5fab26752f98e6966cd407e31db69, regionState=OPEN, openSeqNum=2, regionLocation=d73269e94d72,33145,1732506072477 2024-11-25T03:41:16,180 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for 6aed1980a8b63b44c776d0990f923aa7 2024-11-25T03:41:16,180 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/3b442507eaac3b5222a3c4f97ff80208 2024-11-25T03:41:16,181 INFO [StoreOpener-706facb174cc9547b71f08aa672ddcee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 706facb174cc9547b71f08aa672ddcee 2024-11-25T03:41:16,181 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1048): stopping wal replay for 3b442507eaac3b5222a3c4f97ff80208 2024-11-25T03:41:16,181 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1060): Cleaning up temporary data for 3b442507eaac3b5222a3c4f97ff80208 2024-11-25T03:41:16,182 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/6aed1980a8b63b44c776d0990f923aa7 2024-11-25T03:41:16,183 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/6aed1980a8b63b44c776d0990f923aa7 2024-11-25T03:41:16,184 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for 6aed1980a8b63b44c776d0990f923aa7 2024-11-25T03:41:16,184 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for 6aed1980a8b63b44c776d0990f923aa7 2024-11-25T03:41:16,184 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1093): writing seq id for 3b442507eaac3b5222a3c4f97ff80208 2024-11-25T03:41:16,185 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 85a5fab26752f98e6966cd407e31db69, server=d73269e94d72,33145,1732506072477 because future has completed 2024-11-25T03:41:16,186 INFO [StoreOpener-706facb174cc9547b71f08aa672ddcee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 706facb174cc9547b71f08aa672ddcee columnFamilyName family 2024-11-25T03:41:16,186 DEBUG [StoreOpener-706facb174cc9547b71f08aa672ddcee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:41:16,187 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for 6aed1980a8b63b44c776d0990f923aa7 2024-11-25T03:41:16,187 INFO [StoreOpener-706facb174cc9547b71f08aa672ddcee-1 {}] regionserver.HStore(327): Store=706facb174cc9547b71f08aa672ddcee/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:41:16,188 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 706facb174cc9547b71f08aa672ddcee 2024-11-25T03:41:16,189 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/706facb174cc9547b71f08aa672ddcee 2024-11-25T03:41:16,189 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=29 2024-11-25T03:41:16,190 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure df71ad9490ccea77dfed73e590672db8, server=d73269e94d72,38397,1732506072528 in 396 msec 2024-11-25T03:41:16,190 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/706facb174cc9547b71f08aa672ddcee 2024-11-25T03:41:16,190 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 706facb174cc9547b71f08aa672ddcee 2024-11-25T03:41:16,190 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 706facb174cc9547b71f08aa672ddcee 2024-11-25T03:41:16,192 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/3b442507eaac3b5222a3c4f97ff80208/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:41:16,193 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=df71ad9490ccea77dfed73e590672db8, ASSIGN in 573 msec 2024-11-25T03:41:16,193 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 706facb174cc9547b71f08aa672ddcee 2024-11-25T03:41:16,193 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1114): Opened 3b442507eaac3b5222a3c4f97ff80208; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67728775, jitterRate=0.009237393736839294}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:41:16,193 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3b442507eaac3b5222a3c4f97ff80208 2024-11-25T03:41:16,193 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1006): Region open journal for 3b442507eaac3b5222a3c4f97ff80208: Running coprocessor pre-open hook at 1732506076169Writing region info on filesystem at 1732506076169Initializing all the Stores at 1732506076172 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506076172Cleaning up temporary data from old regions at 1732506076181 (+9 ms)Running coprocessor post-open hooks at 1732506076193 (+12 ms)Region opened successfully at 1732506076193 2024-11-25T03:41:16,194 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1732506074951.3b442507eaac3b5222a3c4f97ff80208., pid=48, masterSystemTime=1732506076009 2024-11-25T03:41:16,196 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/706facb174cc9547b71f08aa672ddcee/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:41:16,197 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2266): Finished post open deploy task for test,,1732506074951.3b442507eaac3b5222a3c4f97ff80208. 2024-11-25T03:41:16,197 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(153): Opened test,,1732506074951.3b442507eaac3b5222a3c4f97ff80208. 2024-11-25T03:41:16,197 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open test,qqq,1732506074951.922ebf14d68aba40da3a2e99777c777b. 2024-11-25T03:41:16,197 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => 922ebf14d68aba40da3a2e99777c777b, NAME => 'test,qqq,1732506074951.922ebf14d68aba40da3a2e99777c777b.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-11-25T03:41:16,198 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 922ebf14d68aba40da3a2e99777c777b 2024-11-25T03:41:16,198 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated test,qqq,1732506074951.922ebf14d68aba40da3a2e99777c777b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:41:16,198 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for 922ebf14d68aba40da3a2e99777c777b 2024-11-25T03:41:16,198 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for 922ebf14d68aba40da3a2e99777c777b 2024-11-25T03:41:16,199 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 706facb174cc9547b71f08aa672ddcee; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72615021, jitterRate=0.08204813301563263}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:41:16,199 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 706facb174cc9547b71f08aa672ddcee 2024-11-25T03:41:16,199 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 706facb174cc9547b71f08aa672ddcee: Running coprocessor pre-open hook at 1732506076175Writing region info on filesystem at 1732506076175Initializing all the Stores at 1732506076179 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506076179Cleaning up temporary data from old regions at 1732506076190 (+11 ms)Running coprocessor post-open hooks at 1732506076199 (+9 ms)Region opened successfully at 1732506076199 2024-11-25T03:41:16,201 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1732506074951.706facb174cc9547b71f08aa672ddcee., pid=37, masterSystemTime=1732506075943 2024-11-25T03:41:16,201 INFO [StoreOpener-922ebf14d68aba40da3a2e99777c777b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 922ebf14d68aba40da3a2e99777c777b 2024-11-25T03:41:16,201 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=8 2024-11-25T03:41:16,201 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 85a5fab26752f98e6966cd407e31db69, server=d73269e94d72,33145,1732506072477 in 237 msec 2024-11-25T03:41:16,202 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/6aed1980a8b63b44c776d0990f923aa7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:41:16,203 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened 6aed1980a8b63b44c776d0990f923aa7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73757617, jitterRate=0.09907414019107819}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:41:16,203 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6aed1980a8b63b44c776d0990f923aa7 2024-11-25T03:41:16,203 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for 6aed1980a8b63b44c776d0990f923aa7: Running coprocessor pre-open hook at 1732506076172Writing region info on filesystem at 1732506076172Initializing all the Stores at 1732506076176 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506076176Cleaning up temporary data from old regions at 1732506076184 (+8 ms)Running coprocessor post-open hooks at 1732506076203 (+19 ms)Region opened successfully at 1732506076203 2024-11-25T03:41:16,204 INFO [StoreOpener-922ebf14d68aba40da3a2e99777c777b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 922ebf14d68aba40da3a2e99777c777b columnFamilyName family 2024-11-25T03:41:16,204 DEBUG [StoreOpener-922ebf14d68aba40da3a2e99777c777b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:41:16,204 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for test,uuu,1732506074951.6aed1980a8b63b44c776d0990f923aa7., pid=45, masterSystemTime=1732506076057 2024-11-25T03:41:16,205 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1732506074951.706facb174cc9547b71f08aa672ddcee. 2024-11-25T03:41:16,205 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened test,vvv,1732506074951.706facb174cc9547b71f08aa672ddcee. 2024-11-25T03:41:16,205 INFO [StoreOpener-922ebf14d68aba40da3a2e99777c777b-1 {}] regionserver.HStore(327): Store=922ebf14d68aba40da3a2e99777c777b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:41:16,205 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open test,ppp,1732506074951.2fc93e882c2b7c5acdb53e9cf1071dc8. 2024-11-25T03:41:16,205 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for 922ebf14d68aba40da3a2e99777c777b 2024-11-25T03:41:16,205 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 2fc93e882c2b7c5acdb53e9cf1071dc8, NAME => 'test,ppp,1732506074951.2fc93e882c2b7c5acdb53e9cf1071dc8.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-25T03:41:16,205 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 2fc93e882c2b7c5acdb53e9cf1071dc8 2024-11-25T03:41:16,206 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated test,ppp,1732506074951.2fc93e882c2b7c5acdb53e9cf1071dc8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:41:16,206 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 2fc93e882c2b7c5acdb53e9cf1071dc8 2024-11-25T03:41:16,206 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 2fc93e882c2b7c5acdb53e9cf1071dc8 2024-11-25T03:41:16,206 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/922ebf14d68aba40da3a2e99777c777b 2024-11-25T03:41:16,207 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/922ebf14d68aba40da3a2e99777c777b 2024-11-25T03:41:16,207 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=85a5fab26752f98e6966cd407e31db69, ASSIGN in 584 msec 2024-11-25T03:41:16,207 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for 922ebf14d68aba40da3a2e99777c777b 2024-11-25T03:41:16,207 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for 922ebf14d68aba40da3a2e99777c777b 2024-11-25T03:41:16,208 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for test,uuu,1732506074951.6aed1980a8b63b44c776d0990f923aa7. 2024-11-25T03:41:16,208 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened test,uuu,1732506074951.6aed1980a8b63b44c776d0990f923aa7. 2024-11-25T03:41:16,208 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open test,rrr,1732506074951.745356a682ec14d9f6e2066d456cab06. 2024-11-25T03:41:16,208 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => 745356a682ec14d9f6e2066d456cab06, NAME => 'test,rrr,1732506074951.745356a682ec14d9f6e2066d456cab06.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-25T03:41:16,209 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 745356a682ec14d9f6e2066d456cab06 2024-11-25T03:41:16,209 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated test,rrr,1732506074951.745356a682ec14d9f6e2066d456cab06.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:41:16,209 INFO [StoreOpener-2fc93e882c2b7c5acdb53e9cf1071dc8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 2fc93e882c2b7c5acdb53e9cf1071dc8 2024-11-25T03:41:16,209 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for 745356a682ec14d9f6e2066d456cab06 2024-11-25T03:41:16,209 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for 745356a682ec14d9f6e2066d456cab06 2024-11-25T03:41:16,210 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for 922ebf14d68aba40da3a2e99777c777b 2024-11-25T03:41:16,210 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=6aed1980a8b63b44c776d0990f923aa7, regionState=OPEN, openSeqNum=2, regionLocation=d73269e94d72,33145,1732506072477 2024-11-25T03:41:16,211 INFO [StoreOpener-2fc93e882c2b7c5acdb53e9cf1071dc8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2fc93e882c2b7c5acdb53e9cf1071dc8 columnFamilyName family 2024-11-25T03:41:16,211 DEBUG [StoreOpener-2fc93e882c2b7c5acdb53e9cf1071dc8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:41:16,212 INFO [StoreOpener-2fc93e882c2b7c5acdb53e9cf1071dc8-1 {}] regionserver.HStore(327): Store=2fc93e882c2b7c5acdb53e9cf1071dc8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:41:16,212 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 2fc93e882c2b7c5acdb53e9cf1071dc8 2024-11-25T03:41:16,212 INFO [StoreOpener-745356a682ec14d9f6e2066d456cab06-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 745356a682ec14d9f6e2066d456cab06 2024-11-25T03:41:16,214 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/2fc93e882c2b7c5acdb53e9cf1071dc8 2024-11-25T03:41:16,214 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/2fc93e882c2b7c5acdb53e9cf1071dc8 2024-11-25T03:41:16,215 INFO [StoreOpener-745356a682ec14d9f6e2066d456cab06-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 745356a682ec14d9f6e2066d456cab06 columnFamilyName family 2024-11-25T03:41:16,215 DEBUG [StoreOpener-745356a682ec14d9f6e2066d456cab06-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:41:16,215 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 2fc93e882c2b7c5acdb53e9cf1071dc8 2024-11-25T03:41:16,215 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6aed1980a8b63b44c776d0990f923aa7, server=d73269e94d72,33145,1732506072477 because future has completed 2024-11-25T03:41:16,215 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 2fc93e882c2b7c5acdb53e9cf1071dc8 2024-11-25T03:41:16,216 INFO [StoreOpener-745356a682ec14d9f6e2066d456cab06-1 {}] regionserver.HStore(327): Store=745356a682ec14d9f6e2066d456cab06/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:41:16,216 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for 745356a682ec14d9f6e2066d456cab06 2024-11-25T03:41:16,218 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/745356a682ec14d9f6e2066d456cab06 2024-11-25T03:41:16,218 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/922ebf14d68aba40da3a2e99777c777b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:41:16,218 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/745356a682ec14d9f6e2066d456cab06 2024-11-25T03:41:16,218 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened 922ebf14d68aba40da3a2e99777c777b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60294079, jitterRate=-0.1015482097864151}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:41:16,218 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 922ebf14d68aba40da3a2e99777c777b 2024-11-25T03:41:16,219 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for 922ebf14d68aba40da3a2e99777c777b: Running coprocessor pre-open hook at 1732506076198Writing region info on filesystem at 1732506076198Initializing all the Stores at 1732506076200 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506076200Cleaning up temporary data from old regions at 1732506076207 (+7 ms)Running coprocessor post-open hooks at 1732506076218 (+11 ms)Region opened successfully at 1732506076218 2024-11-25T03:41:16,219 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for 745356a682ec14d9f6e2066d456cab06 2024-11-25T03:41:16,219 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for 745356a682ec14d9f6e2066d456cab06 2024-11-25T03:41:16,220 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 2fc93e882c2b7c5acdb53e9cf1071dc8 2024-11-25T03:41:16,220 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40325 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=d73269e94d72,38397,1732506072528, table=test, region=8a7d4164efdaed87f07b6f76a94819b3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-25T03:41:16,221 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for test,qqq,1732506074951.922ebf14d68aba40da3a2e99777c777b., pid=40, masterSystemTime=1732506076009 2024-11-25T03:41:16,224 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for 745356a682ec14d9f6e2066d456cab06 2024-11-25T03:41:16,225 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for test,qqq,1732506074951.922ebf14d68aba40da3a2e99777c777b. 2024-11-25T03:41:16,225 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened test,qqq,1732506074951.922ebf14d68aba40da3a2e99777c777b. 2024-11-25T03:41:16,226 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open test,www,1732506074951.2b224c7c31e6ff664ce67ac884f573d7. 2024-11-25T03:41:16,226 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => 2b224c7c31e6ff664ce67ac884f573d7, NAME => 'test,www,1732506074951.2b224c7c31e6ff664ce67ac884f573d7.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-11-25T03:41:16,226 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 2b224c7c31e6ff664ce67ac884f573d7 2024-11-25T03:41:16,226 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated test,www,1732506074951.2b224c7c31e6ff664ce67ac884f573d7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:41:16,227 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for 2b224c7c31e6ff664ce67ac884f573d7 2024-11-25T03:41:16,227 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for 2b224c7c31e6ff664ce67ac884f573d7 2024-11-25T03:41:16,228 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/2fc93e882c2b7c5acdb53e9cf1071dc8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:41:16,228 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=25 2024-11-25T03:41:16,228 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure 6aed1980a8b63b44c776d0990f923aa7, server=d73269e94d72,33145,1732506072477 in 317 msec 2024-11-25T03:41:16,229 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 2fc93e882c2b7c5acdb53e9cf1071dc8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71876274, jitterRate=0.07103994488716125}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:41:16,229 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2fc93e882c2b7c5acdb53e9cf1071dc8 2024-11-25T03:41:16,229 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 2fc93e882c2b7c5acdb53e9cf1071dc8: Running coprocessor pre-open hook at 1732506076206Writing region info on filesystem at 1732506076206Initializing all the Stores at 1732506076208 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506076208Cleaning up temporary data from old regions at 1732506076215 (+7 ms)Running coprocessor post-open hooks at 1732506076229 (+14 ms)Region opened successfully at 1732506076229 2024-11-25T03:41:16,230 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/745356a682ec14d9f6e2066d456cab06/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:41:16,230 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1732506074951.2fc93e882c2b7c5acdb53e9cf1071dc8., pid=32, masterSystemTime=1732506075943 2024-11-25T03:41:16,230 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened 745356a682ec14d9f6e2066d456cab06; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59717020, jitterRate=-0.11014705896377563}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:41:16,231 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 745356a682ec14d9f6e2066d456cab06 2024-11-25T03:41:16,231 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for 745356a682ec14d9f6e2066d456cab06: Running coprocessor pre-open hook at 1732506076209Writing region info on filesystem at 1732506076209Initializing all the Stores at 1732506076211 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506076211Cleaning up temporary data from old regions at 1732506076219 (+8 ms)Running coprocessor post-open hooks at 1732506076231 (+12 ms)Region opened successfully at 1732506076231 2024-11-25T03:41:16,232 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1732506074951.745356a682ec14d9f6e2066d456cab06., pid=46, masterSystemTime=1732506076057 2024-11-25T03:41:16,232 INFO [StoreOpener-2b224c7c31e6ff664ce67ac884f573d7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 2b224c7c31e6ff664ce67ac884f573d7 2024-11-25T03:41:16,233 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6aed1980a8b63b44c776d0990f923aa7, ASSIGN in 612 msec 2024-11-25T03:41:16,234 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ppp,1732506074951.2fc93e882c2b7c5acdb53e9cf1071dc8. 2024-11-25T03:41:16,234 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened test,ppp,1732506074951.2fc93e882c2b7c5acdb53e9cf1071dc8. 2024-11-25T03:41:16,235 INFO [StoreOpener-2b224c7c31e6ff664ce67ac884f573d7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2b224c7c31e6ff664ce67ac884f573d7 columnFamilyName family 2024-11-25T03:41:16,235 DEBUG [StoreOpener-2b224c7c31e6ff664ce67ac884f573d7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:41:16,236 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=2fc93e882c2b7c5acdb53e9cf1071dc8, regionState=OPEN, openSeqNum=2, regionLocation=d73269e94d72,38397,1732506072528 2024-11-25T03:41:16,236 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1732506074951.745356a682ec14d9f6e2066d456cab06. 2024-11-25T03:41:16,236 INFO [StoreOpener-2b224c7c31e6ff664ce67ac884f573d7-1 {}] regionserver.HStore(327): Store=2b224c7c31e6ff664ce67ac884f573d7/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:41:16,236 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened test,rrr,1732506074951.745356a682ec14d9f6e2066d456cab06. 2024-11-25T03:41:16,236 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for 2b224c7c31e6ff664ce67ac884f573d7 2024-11-25T03:41:16,236 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(132): Open test,kkk,1732506074951.a4b8682bf62ac734aa4174eba4ee1ddc. 2024-11-25T03:41:16,236 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7752): Opening region: {ENCODED => a4b8682bf62ac734aa4174eba4ee1ddc, NAME => 'test,kkk,1732506074951.a4b8682bf62ac734aa4174eba4ee1ddc.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-11-25T03:41:16,237 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test a4b8682bf62ac734aa4174eba4ee1ddc 2024-11-25T03:41:16,237 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(898): Instantiated test,kkk,1732506074951.a4b8682bf62ac734aa4174eba4ee1ddc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:41:16,237 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7794): checking encryption for a4b8682bf62ac734aa4174eba4ee1ddc 2024-11-25T03:41:16,237 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7797): checking classloading for a4b8682bf62ac734aa4174eba4ee1ddc 2024-11-25T03:41:16,237 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/2b224c7c31e6ff664ce67ac884f573d7 2024-11-25T03:41:16,239 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/2b224c7c31e6ff664ce67ac884f573d7 2024-11-25T03:41:16,239 INFO [StoreOpener-a4b8682bf62ac734aa4174eba4ee1ddc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region a4b8682bf62ac734aa4174eba4ee1ddc 2024-11-25T03:41:16,239 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for 2b224c7c31e6ff664ce67ac884f573d7 2024-11-25T03:41:16,240 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for 2b224c7c31e6ff664ce67ac884f573d7 2024-11-25T03:41:16,241 INFO [StoreOpener-a4b8682bf62ac734aa4174eba4ee1ddc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4b8682bf62ac734aa4174eba4ee1ddc columnFamilyName family 2024-11-25T03:41:16,242 DEBUG [StoreOpener-a4b8682bf62ac734aa4174eba4ee1ddc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:41:16,242 INFO [StoreOpener-a4b8682bf62ac734aa4174eba4ee1ddc-1 {}] regionserver.HStore(327): Store=a4b8682bf62ac734aa4174eba4ee1ddc/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:41:16,243 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1038): replaying wal for a4b8682bf62ac734aa4174eba4ee1ddc 2024-11-25T03:41:16,243 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=745356a682ec14d9f6e2066d456cab06, regionState=OPEN, openSeqNum=2, regionLocation=d73269e94d72,33145,1732506072477 2024-11-25T03:41:16,243 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for 2b224c7c31e6ff664ce67ac884f573d7 2024-11-25T03:41:16,243 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2fc93e882c2b7c5acdb53e9cf1071dc8, server=d73269e94d72,38397,1732506072528 because future has completed 2024-11-25T03:41:16,244 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/a4b8682bf62ac734aa4174eba4ee1ddc 2024-11-25T03:41:16,245 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40325 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=d73269e94d72,33145,1732506072477, table=test, region=745356a682ec14d9f6e2066d456cab06. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-25T03:41:16,246 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/a4b8682bf62ac734aa4174eba4ee1ddc 2024-11-25T03:41:16,247 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1048): stopping wal replay for a4b8682bf62ac734aa4174eba4ee1ddc 2024-11-25T03:41:16,247 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1060): Cleaning up temporary data for a4b8682bf62ac734aa4174eba4ee1ddc 2024-11-25T03:41:16,247 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 745356a682ec14d9f6e2066d456cab06, server=d73269e94d72,33145,1732506072477 because future has completed 2024-11-25T03:41:16,249 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/2b224c7c31e6ff664ce67ac884f573d7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:41:16,250 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): writing seq id for a4b8682bf62ac734aa4174eba4ee1ddc 2024-11-25T03:41:16,250 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened 2b224c7c31e6ff664ce67ac884f573d7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72510600, jitterRate=0.08049213886260986}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:41:16,250 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2b224c7c31e6ff664ce67ac884f573d7 2024-11-25T03:41:16,250 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for 2b224c7c31e6ff664ce67ac884f573d7: Running coprocessor pre-open hook at 1732506076227Writing region info on filesystem at 1732506076227Initializing all the Stores at 1732506076229 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506076229Cleaning up temporary data from old regions at 1732506076240 (+11 ms)Running coprocessor post-open hooks at 1732506076250 (+10 ms)Region opened successfully at 1732506076250 2024-11-25T03:41:16,252 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for test,www,1732506074951.2b224c7c31e6ff664ce67ac884f573d7., pid=41, masterSystemTime=1732506076009 2024-11-25T03:41:16,254 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=20 2024-11-25T03:41:16,254 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 2fc93e882c2b7c5acdb53e9cf1071dc8, server=d73269e94d72,38397,1732506072528 in 452 msec 2024-11-25T03:41:16,255 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for test,www,1732506074951.2b224c7c31e6ff664ce67ac884f573d7. 2024-11-25T03:41:16,256 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened test,www,1732506074951.2b224c7c31e6ff664ce67ac884f573d7. 2024-11-25T03:41:16,256 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(132): Open test,jjj,1732506074951.f5f64cb2042c701322ac00e312f65f16. 2024-11-25T03:41:16,256 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7752): Opening region: {ENCODED => f5f64cb2042c701322ac00e312f65f16, NAME => 'test,jjj,1732506074951.f5f64cb2042c701322ac00e312f65f16.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-25T03:41:16,257 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f5f64cb2042c701322ac00e312f65f16 2024-11-25T03:41:16,257 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(898): Instantiated test,jjj,1732506074951.f5f64cb2042c701322ac00e312f65f16.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:41:16,257 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7794): checking encryption for f5f64cb2042c701322ac00e312f65f16 2024-11-25T03:41:16,257 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7797): checking classloading for f5f64cb2042c701322ac00e312f65f16 2024-11-25T03:41:16,257 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/a4b8682bf62ac734aa4174eba4ee1ddc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:41:16,258 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=2fc93e882c2b7c5acdb53e9cf1071dc8, ASSIGN in 637 msec 2024-11-25T03:41:16,258 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1114): Opened a4b8682bf62ac734aa4174eba4ee1ddc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69844196, jitterRate=0.0407596230506897}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:41:16,258 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a4b8682bf62ac734aa4174eba4ee1ddc 2024-11-25T03:41:16,259 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1006): Region open journal for a4b8682bf62ac734aa4174eba4ee1ddc: Running coprocessor pre-open hook at 1732506076237Writing region info on filesystem at 1732506076237Initializing all the Stores at 1732506076239 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506076239Cleaning up temporary data from old regions at 1732506076247 (+8 ms)Running coprocessor post-open hooks at 1732506076258 (+11 ms)Region opened successfully at 1732506076259 (+1 ms) 2024-11-25T03:41:16,260 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=2b224c7c31e6ff664ce67ac884f573d7, regionState=OPEN, openSeqNum=2, regionLocation=d73269e94d72,35981,1732506072374 2024-11-25T03:41:16,260 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2236): Post open deploy tasks for test,kkk,1732506074951.a4b8682bf62ac734aa4174eba4ee1ddc., pid=52, masterSystemTime=1732506076057 2024-11-25T03:41:16,260 INFO [StoreOpener-f5f64cb2042c701322ac00e312f65f16-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f5f64cb2042c701322ac00e312f65f16 2024-11-25T03:41:16,263 INFO [StoreOpener-f5f64cb2042c701322ac00e312f65f16-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f5f64cb2042c701322ac00e312f65f16 columnFamilyName family 2024-11-25T03:41:16,263 DEBUG [StoreOpener-f5f64cb2042c701322ac00e312f65f16-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:41:16,263 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2266): Finished post open deploy task for test,kkk,1732506074951.a4b8682bf62ac734aa4174eba4ee1ddc. 2024-11-25T03:41:16,263 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b224c7c31e6ff664ce67ac884f573d7, server=d73269e94d72,35981,1732506072374 because future has completed 2024-11-25T03:41:16,263 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(153): Opened test,kkk,1732506074951.a4b8682bf62ac734aa4174eba4ee1ddc. 2024-11-25T03:41:16,263 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open test,bbb,1732506074951.c73ee0e3b504288792edda3ec9a26049. 2024-11-25T03:41:16,263 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => c73ee0e3b504288792edda3ec9a26049, NAME => 'test,bbb,1732506074951.c73ee0e3b504288792edda3ec9a26049.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-11-25T03:41:16,264 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c73ee0e3b504288792edda3ec9a26049 2024-11-25T03:41:16,264 INFO [StoreOpener-f5f64cb2042c701322ac00e312f65f16-1 {}] regionserver.HStore(327): Store=f5f64cb2042c701322ac00e312f65f16/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:41:16,264 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated test,bbb,1732506074951.c73ee0e3b504288792edda3ec9a26049.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:41:16,264 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for c73ee0e3b504288792edda3ec9a26049 2024-11-25T03:41:16,264 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1038): replaying wal for f5f64cb2042c701322ac00e312f65f16 2024-11-25T03:41:16,264 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for c73ee0e3b504288792edda3ec9a26049 2024-11-25T03:41:16,265 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/f5f64cb2042c701322ac00e312f65f16 2024-11-25T03:41:16,266 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/f5f64cb2042c701322ac00e312f65f16 2024-11-25T03:41:16,266 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1048): stopping wal replay for f5f64cb2042c701322ac00e312f65f16 2024-11-25T03:41:16,266 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1060): Cleaning up temporary data for f5f64cb2042c701322ac00e312f65f16 2024-11-25T03:41:16,267 INFO [StoreOpener-c73ee0e3b504288792edda3ec9a26049-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c73ee0e3b504288792edda3ec9a26049 2024-11-25T03:41:16,269 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=22 2024-11-25T03:41:16,269 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1093): writing seq id for f5f64cb2042c701322ac00e312f65f16 2024-11-25T03:41:16,269 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure 745356a682ec14d9f6e2066d456cab06, server=d73269e94d72,33145,1732506072477 in 354 msec 2024-11-25T03:41:16,269 INFO [StoreOpener-c73ee0e3b504288792edda3ec9a26049-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c73ee0e3b504288792edda3ec9a26049 columnFamilyName family 2024-11-25T03:41:16,269 DEBUG [StoreOpener-c73ee0e3b504288792edda3ec9a26049-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:41:16,270 INFO [StoreOpener-c73ee0e3b504288792edda3ec9a26049-1 {}] regionserver.HStore(327): Store=c73ee0e3b504288792edda3ec9a26049/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:41:16,271 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for c73ee0e3b504288792edda3ec9a26049 2024-11-25T03:41:16,272 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/c73ee0e3b504288792edda3ec9a26049 2024-11-25T03:41:16,272 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/c73ee0e3b504288792edda3ec9a26049 2024-11-25T03:41:16,273 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for c73ee0e3b504288792edda3ec9a26049 2024-11-25T03:41:16,273 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for c73ee0e3b504288792edda3ec9a26049 2024-11-25T03:41:16,273 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/f5f64cb2042c701322ac00e312f65f16/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:41:16,274 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1114): Opened f5f64cb2042c701322ac00e312f65f16; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59769677, jitterRate=-0.1093624085187912}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:41:16,274 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f5f64cb2042c701322ac00e312f65f16 2024-11-25T03:41:16,274 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=745356a682ec14d9f6e2066d456cab06, ASSIGN in 652 msec 2024-11-25T03:41:16,274 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1006): Region open journal for f5f64cb2042c701322ac00e312f65f16: Running coprocessor pre-open hook at 1732506076257Writing region info on filesystem at 1732506076257Initializing all the Stores at 1732506076259 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506076259Cleaning up temporary data from old regions at 1732506076266 (+7 ms)Running coprocessor post-open hooks at 1732506076274 (+8 ms)Region opened successfully at 1732506076274 2024-11-25T03:41:16,275 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2236): Post open deploy tasks for test,jjj,1732506074951.f5f64cb2042c701322ac00e312f65f16., pid=50, masterSystemTime=1732506076009 2024-11-25T03:41:16,276 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for c73ee0e3b504288792edda3ec9a26049 2024-11-25T03:41:16,279 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2266): Finished post open deploy task for test,jjj,1732506074951.f5f64cb2042c701322ac00e312f65f16. 2024-11-25T03:41:16,279 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(153): Opened test,jjj,1732506074951.f5f64cb2042c701322ac00e312f65f16. 2024-11-25T03:41:16,280 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/c73ee0e3b504288792edda3ec9a26049/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:41:16,280 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=27 2024-11-25T03:41:16,280 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure 2b224c7c31e6ff664ce67ac884f573d7, server=d73269e94d72,35981,1732506072374 in 407 msec 2024-11-25T03:41:16,280 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened c73ee0e3b504288792edda3ec9a26049; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73710004, jitterRate=0.09836465120315552}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:41:16,280 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c73ee0e3b504288792edda3ec9a26049 2024-11-25T03:41:16,281 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for c73ee0e3b504288792edda3ec9a26049: Running coprocessor pre-open hook at 1732506076264Writing region info on filesystem at 1732506076264Initializing all the Stores at 1732506076266 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506076266Cleaning up temporary data from old regions at 1732506076273 (+7 ms)Running coprocessor post-open hooks at 1732506076280 (+7 ms)Region opened successfully at 1732506076281 (+1 ms) 2024-11-25T03:41:16,281 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for test,bbb,1732506074951.c73ee0e3b504288792edda3ec9a26049., pid=53, masterSystemTime=1732506076057 2024-11-25T03:41:16,283 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=2b224c7c31e6ff664ce67ac884f573d7, ASSIGN in 663 msec 2024-11-25T03:41:16,284 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for test,bbb,1732506074951.c73ee0e3b504288792edda3ec9a26049. 2024-11-25T03:41:16,284 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened test,bbb,1732506074951.c73ee0e3b504288792edda3ec9a26049. 2024-11-25T03:41:16,284 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(132): Open test,ooo,1732506074951.5da71c9c32aa33cf6f067ea6d82ae84f. 2024-11-25T03:41:16,284 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=f5f64cb2042c701322ac00e312f65f16, regionState=OPEN, openSeqNum=2, regionLocation=d73269e94d72,35981,1732506072374 2024-11-25T03:41:16,284 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7752): Opening region: {ENCODED => 5da71c9c32aa33cf6f067ea6d82ae84f, NAME => 'test,ooo,1732506074951.5da71c9c32aa33cf6f067ea6d82ae84f.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-11-25T03:41:16,285 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 5da71c9c32aa33cf6f067ea6d82ae84f 2024-11-25T03:41:16,285 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(898): Instantiated test,ooo,1732506074951.5da71c9c32aa33cf6f067ea6d82ae84f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:41:16,285 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7794): checking encryption for 5da71c9c32aa33cf6f067ea6d82ae84f 2024-11-25T03:41:16,285 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7797): checking classloading for 5da71c9c32aa33cf6f067ea6d82ae84f 2024-11-25T03:41:16,287 INFO [StoreOpener-5da71c9c32aa33cf6f067ea6d82ae84f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 5da71c9c32aa33cf6f067ea6d82ae84f 2024-11-25T03:41:16,288 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5f64cb2042c701322ac00e312f65f16, server=d73269e94d72,35981,1732506072374 because future has completed 2024-11-25T03:41:16,288 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=c73ee0e3b504288792edda3ec9a26049, regionState=OPEN, openSeqNum=2, regionLocation=d73269e94d72,33145,1732506072477 2024-11-25T03:41:16,290 INFO [StoreOpener-5da71c9c32aa33cf6f067ea6d82ae84f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5da71c9c32aa33cf6f067ea6d82ae84f columnFamilyName family 2024-11-25T03:41:16,290 DEBUG [StoreOpener-5da71c9c32aa33cf6f067ea6d82ae84f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:41:16,291 INFO [StoreOpener-5da71c9c32aa33cf6f067ea6d82ae84f-1 {}] regionserver.HStore(327): Store=5da71c9c32aa33cf6f067ea6d82ae84f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:41:16,291 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1038): replaying wal for 5da71c9c32aa33cf6f067ea6d82ae84f 2024-11-25T03:41:16,291 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure c73ee0e3b504288792edda3ec9a26049, server=d73269e94d72,33145,1732506072477 because future has completed 2024-11-25T03:41:16,292 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/5da71c9c32aa33cf6f067ea6d82ae84f 2024-11-25T03:41:16,293 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/5da71c9c32aa33cf6f067ea6d82ae84f 2024-11-25T03:41:16,293 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1048): stopping wal replay for 5da71c9c32aa33cf6f067ea6d82ae84f 2024-11-25T03:41:16,293 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1060): Cleaning up temporary data for 5da71c9c32aa33cf6f067ea6d82ae84f 2024-11-25T03:41:16,295 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=14 2024-11-25T03:41:16,295 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure f5f64cb2042c701322ac00e312f65f16, server=d73269e94d72,35981,1732506072374 in 360 msec 2024-11-25T03:41:16,296 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): writing seq id for 5da71c9c32aa33cf6f067ea6d82ae84f 2024-11-25T03:41:16,299 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f5f64cb2042c701322ac00e312f65f16, ASSIGN in 679 msec 2024-11-25T03:41:16,299 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/5da71c9c32aa33cf6f067ea6d82ae84f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:41:16,300 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1114): Opened 5da71c9c32aa33cf6f067ea6d82ae84f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61614125, jitterRate=-0.08187799155712128}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:41:16,300 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5da71c9c32aa33cf6f067ea6d82ae84f 2024-11-25T03:41:16,301 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1006): Region open journal for 5da71c9c32aa33cf6f067ea6d82ae84f: Running coprocessor pre-open hook at 1732506076285Writing region info on filesystem at 1732506076285Initializing all the Stores at 1732506076287 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506076287Cleaning up temporary data from old regions at 1732506076293 (+6 ms)Running coprocessor post-open hooks at 1732506076300 (+7 ms)Region opened successfully at 1732506076300 2024-11-25T03:41:16,301 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ooo,1732506074951.5da71c9c32aa33cf6f067ea6d82ae84f., pid=51, masterSystemTime=1732506076057 2024-11-25T03:41:16,304 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=6 2024-11-25T03:41:16,304 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure c73ee0e3b504288792edda3ec9a26049, server=d73269e94d72,33145,1732506072477 in 349 msec 2024-11-25T03:41:16,305 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ooo,1732506074951.5da71c9c32aa33cf6f067ea6d82ae84f. 2024-11-25T03:41:16,305 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(153): Opened test,ooo,1732506074951.5da71c9c32aa33cf6f067ea6d82ae84f. 2024-11-25T03:41:16,305 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(132): Open test,hhh,1732506074951.5e791add66a4fee50be34aa83a6d502a. 2024-11-25T03:41:16,305 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7752): Opening region: {ENCODED => 5e791add66a4fee50be34aa83a6d502a, NAME => 'test,hhh,1732506074951.5e791add66a4fee50be34aa83a6d502a.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-25T03:41:16,306 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 5e791add66a4fee50be34aa83a6d502a 2024-11-25T03:41:16,306 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(898): Instantiated test,hhh,1732506074951.5e791add66a4fee50be34aa83a6d502a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:41:16,306 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7794): checking encryption for 5e791add66a4fee50be34aa83a6d502a 2024-11-25T03:41:16,306 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7797): checking classloading for 5e791add66a4fee50be34aa83a6d502a 2024-11-25T03:41:16,307 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=c73ee0e3b504288792edda3ec9a26049, ASSIGN in 688 msec 2024-11-25T03:41:16,308 INFO [StoreOpener-5e791add66a4fee50be34aa83a6d502a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 5e791add66a4fee50be34aa83a6d502a 2024-11-25T03:41:16,308 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=5da71c9c32aa33cf6f067ea6d82ae84f, regionState=OPEN, openSeqNum=2, regionLocation=d73269e94d72,33145,1732506072477 2024-11-25T03:41:16,310 INFO [StoreOpener-5e791add66a4fee50be34aa83a6d502a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5e791add66a4fee50be34aa83a6d502a columnFamilyName family 2024-11-25T03:41:16,310 DEBUG [StoreOpener-5e791add66a4fee50be34aa83a6d502a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:41:16,311 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5da71c9c32aa33cf6f067ea6d82ae84f, server=d73269e94d72,33145,1732506072477 because future has completed 2024-11-25T03:41:16,311 INFO [StoreOpener-5e791add66a4fee50be34aa83a6d502a-1 {}] regionserver.HStore(327): Store=5e791add66a4fee50be34aa83a6d502a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:41:16,312 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1038): replaying wal for 5e791add66a4fee50be34aa83a6d502a 2024-11-25T03:41:16,312 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=a4b8682bf62ac734aa4174eba4ee1ddc, regionState=OPEN, openSeqNum=2, regionLocation=d73269e94d72,33145,1732506072477 2024-11-25T03:41:16,313 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/5e791add66a4fee50be34aa83a6d502a 2024-11-25T03:41:16,313 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/5e791add66a4fee50be34aa83a6d502a 2024-11-25T03:41:16,314 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1048): stopping wal replay for 5e791add66a4fee50be34aa83a6d502a 2024-11-25T03:41:16,314 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1060): Cleaning up temporary data for 5e791add66a4fee50be34aa83a6d502a 2024-11-25T03:41:16,315 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure a4b8682bf62ac734aa4174eba4ee1ddc, server=d73269e94d72,33145,1732506072477 because future has completed 2024-11-25T03:41:16,318 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1093): writing seq id for 5e791add66a4fee50be34aa83a6d502a 2024-11-25T03:41:16,318 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=19 2024-11-25T03:41:16,319 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=19, state=SUCCESS, hasLock=false; OpenRegionProcedure 5da71c9c32aa33cf6f067ea6d82ae84f, server=d73269e94d72,33145,1732506072477 in 375 msec 2024-11-25T03:41:16,321 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/5e791add66a4fee50be34aa83a6d502a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:41:16,322 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=5da71c9c32aa33cf6f067ea6d82ae84f, ASSIGN in 702 msec 2024-11-25T03:41:16,323 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1114): Opened 5e791add66a4fee50be34aa83a6d502a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67233924, jitterRate=0.0018635392189025879}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:41:16,324 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5e791add66a4fee50be34aa83a6d502a 2024-11-25T03:41:16,324 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1006): Region open journal for 5e791add66a4fee50be34aa83a6d502a: Running coprocessor pre-open hook at 1732506076306Writing region info on filesystem at 1732506076306Initializing all the Stores at 1732506076307 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506076307Cleaning up temporary data from old regions at 1732506076314 (+7 ms)Running coprocessor post-open hooks at 1732506076324 (+10 ms)Region opened successfully at 1732506076324 2024-11-25T03:41:16,325 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40325 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=d73269e94d72,38397,1732506072528, table=test, region=8a7d4164efdaed87f07b6f76a94819b3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-25T03:41:16,325 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1732506074951.5e791add66a4fee50be34aa83a6d502a., pid=56, masterSystemTime=1732506076057 2024-11-25T03:41:16,328 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2266): Finished post open deploy task for test,hhh,1732506074951.5e791add66a4fee50be34aa83a6d502a. 2024-11-25T03:41:16,328 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=15 2024-11-25T03:41:16,328 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(153): Opened test,hhh,1732506074951.5e791add66a4fee50be34aa83a6d502a. 2024-11-25T03:41:16,328 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure a4b8682bf62ac734aa4174eba4ee1ddc, server=d73269e94d72,33145,1732506072477 in 378 msec 2024-11-25T03:41:16,331 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a4b8682bf62ac734aa4174eba4ee1ddc, ASSIGN in 711 msec 2024-11-25T03:41:16,332 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=5e791add66a4fee50be34aa83a6d502a, regionState=OPEN, openSeqNum=2, regionLocation=d73269e94d72,33145,1732506072477 2024-11-25T03:41:16,335 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5e791add66a4fee50be34aa83a6d502a, server=d73269e94d72,33145,1732506072477 because future has completed 2024-11-25T03:41:16,336 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=922ebf14d68aba40da3a2e99777c777b, regionState=OPEN, openSeqNum=2, regionLocation=d73269e94d72,35981,1732506072374 2024-11-25T03:41:16,339 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 922ebf14d68aba40da3a2e99777c777b, server=d73269e94d72,35981,1732506072374 because future has completed 2024-11-25T03:41:16,341 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=12 2024-11-25T03:41:16,341 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure 5e791add66a4fee50be34aa83a6d502a, server=d73269e94d72,33145,1732506072477 in 370 msec 2024-11-25T03:41:16,343 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=5e791add66a4fee50be34aa83a6d502a, ASSIGN in 725 msec 2024-11-25T03:41:16,348 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=21 2024-11-25T03:41:16,349 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=21, state=SUCCESS, hasLock=false; OpenRegionProcedure 922ebf14d68aba40da3a2e99777c777b, server=d73269e94d72,35981,1732506072374 in 482 msec 2024-11-25T03:41:16,351 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN in 732 msec 2024-11-25T03:41:16,352 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=706facb174cc9547b71f08aa672ddcee, regionState=OPEN, openSeqNum=2, regionLocation=d73269e94d72,38397,1732506072528 2024-11-25T03:41:16,355 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 706facb174cc9547b71f08aa672ddcee, server=d73269e94d72,38397,1732506072528 because future has completed 2024-11-25T03:41:16,355 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=3b442507eaac3b5222a3c4f97ff80208, regionState=OPEN, openSeqNum=2, regionLocation=d73269e94d72,35981,1732506072374 2024-11-25T03:41:16,358 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=48, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,35981,1732506072374 because future has completed 2024-11-25T03:41:16,362 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=26 2024-11-25T03:41:16,362 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 706facb174cc9547b71f08aa672ddcee, server=d73269e94d72,38397,1732506072528 in 526 msec 2024-11-25T03:41:16,363 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=706facb174cc9547b71f08aa672ddcee, ASSIGN in 745 msec 2024-11-25T03:41:16,366 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=5 2024-11-25T03:41:16,366 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,35981,1732506072374 in 442 msec 2024-11-25T03:41:16,368 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN in 750 msec 2024-11-25T03:41:16,369 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=c769ca025a015554efcc98619008be96, regionState=OPEN, openSeqNum=2, regionLocation=d73269e94d72,35981,1732506072374 2024-11-25T03:41:16,372 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=bba1924bf10ad03aeb55628aeffc2363, regionState=OPEN, openSeqNum=2, regionLocation=d73269e94d72,33145,1732506072477 2024-11-25T03:41:16,372 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure c769ca025a015554efcc98619008be96, server=d73269e94d72,35981,1732506072374 because future has completed 2024-11-25T03:41:16,375 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure bba1924bf10ad03aeb55628aeffc2363, server=d73269e94d72,33145,1732506072477 because future has completed 2024-11-25T03:41:16,379 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=30 2024-11-25T03:41:16,379 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure c769ca025a015554efcc98619008be96, server=d73269e94d72,35981,1732506072374 in 500 msec 2024-11-25T03:41:16,381 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=c769ca025a015554efcc98619008be96, ASSIGN in 762 msec 2024-11-25T03:41:16,384 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=10 2024-11-25T03:41:16,385 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure bba1924bf10ad03aeb55628aeffc2363, server=d73269e94d72,33145,1732506072477 in 419 msec 2024-11-25T03:41:16,387 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=bba1924bf10ad03aeb55628aeffc2363, ASSIGN in 768 msec 2024-11-25T03:41:16,390 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=9 2024-11-25T03:41:16,390 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure 8a7d4164efdaed87f07b6f76a94819b3, server=d73269e94d72,38397,1732506072528 in 542 msec 2024-11-25T03:41:16,392 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8a7d4164efdaed87f07b6f76a94819b3, ASSIGN in 773 msec 2024-11-25T03:41:16,393 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=857a6b6999e909a9b73e698c69291205, regionState=OPEN, openSeqNum=2, regionLocation=d73269e94d72,35981,1732506072374 2024-11-25T03:41:16,395 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 857a6b6999e909a9b73e698c69291205, server=d73269e94d72,35981,1732506072374 because future has completed 2024-11-25T03:41:16,399 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=11 2024-11-25T03:41:16,399 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 857a6b6999e909a9b73e698c69291205, server=d73269e94d72,35981,1732506072374 in 470 msec 2024-11-25T03:41:16,402 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=4 2024-11-25T03:41:16,402 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN in 782 msec 2024-11-25T03:41:16,403 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T03:41:16,403 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732506076403"}]},"ts":"1732506076403"} 2024-11-25T03:41:16,406 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLED in hbase:meta 2024-11-25T03:41:16,407 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T03:41:16,413 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test in 1.4490 sec 2024-11-25T03:41:17,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T03:41:17,128 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test completed 2024-11-25T03:41:17,128 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table test get assigned. Timeout = 60000ms 2024-11-25T03:41:17,130 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T03:41:17,155 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table test assigned to meta. Checking AM states. 2024-11-25T03:41:17,156 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T03:41:17,156 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table test assigned. 2024-11-25T03:41:17,157 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T03:41:17,162 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-25T03:41:17,189 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-25T03:41:17,207 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:41:17,210 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46908, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:41:17,216 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:41:17,218 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56218, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:41:17,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40325 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c73ee0e3b504288792edda3ec9a26049 changed from -1.0 to 0.0, refreshing cache 2024-11-25T03:41:17,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40325 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 85a5fab26752f98e6966cd407e31db69 changed from -1.0 to 0.0, refreshing cache 2024-11-25T03:41:17,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40325 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region bba1924bf10ad03aeb55628aeffc2363 changed from -1.0 to 0.0, refreshing cache 2024-11-25T03:41:17,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40325 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 5e791add66a4fee50be34aa83a6d502a changed from -1.0 to 0.0, refreshing cache 2024-11-25T03:41:17,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40325 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region a4b8682bf62ac734aa4174eba4ee1ddc changed from -1.0 to 0.0, refreshing cache 2024-11-25T03:41:17,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40325 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 5da71c9c32aa33cf6f067ea6d82ae84f changed from -1.0 to 0.0, refreshing cache 2024-11-25T03:41:17,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40325 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 745356a682ec14d9f6e2066d456cab06 changed from -1.0 to 0.0, refreshing cache 2024-11-25T03:41:17,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40325 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 6aed1980a8b63b44c776d0990f923aa7 changed from -1.0 to 0.0, refreshing cache 2024-11-25T03:41:17,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40325 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 5f812dacea03120e7730979dc37e38c6 changed from -1.0 to 0.0, refreshing cache 2024-11-25T03:41:17,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40325 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 3b442507eaac3b5222a3c4f97ff80208 changed from -1.0 to 0.0, refreshing cache 2024-11-25T03:41:17,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40325 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 857a6b6999e909a9b73e698c69291205 changed from -1.0 to 0.0, refreshing cache 2024-11-25T03:41:17,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40325 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f5f64cb2042c701322ac00e312f65f16 changed from -1.0 to 0.0, refreshing cache 2024-11-25T03:41:17,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40325 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 4705fa4ae38907dd8b9783be2dff29da changed from -1.0 to 0.0, refreshing cache 2024-11-25T03:41:17,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40325 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 922ebf14d68aba40da3a2e99777c777b changed from -1.0 to 0.0, refreshing cache 2024-11-25T03:41:17,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40325 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 8b5ae4b02f22b24f40505bb660957004 changed from -1.0 to 0.0, refreshing cache 2024-11-25T03:41:17,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40325 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 2b224c7c31e6ff664ce67ac884f573d7 changed from -1.0 to 0.0, refreshing cache 2024-11-25T03:41:17,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40325 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c769ca025a015554efcc98619008be96 changed from -1.0 to 0.0, refreshing cache 2024-11-25T03:41:17,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40325 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 5e3a490d70b339ac73875ea7638f934f changed from -1.0 to 0.0, refreshing cache 2024-11-25T03:41:17,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40325 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 8a7d4164efdaed87f07b6f76a94819b3 changed from -1.0 to 0.0, refreshing cache 2024-11-25T03:41:17,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40325 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 4c0b379dd00b53bde73bca0e12f321ce changed from -1.0 to 0.0, refreshing cache 2024-11-25T03:41:17,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40325 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region eb9ff3be6d639a626d88387efa9120fd changed from -1.0 to 0.0, refreshing cache 2024-11-25T03:41:17,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40325 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 0c260efb5b2a8a6137010d439efc8d47 changed from -1.0 to 0.0, refreshing cache 2024-11-25T03:41:17,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40325 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 2fc93e882c2b7c5acdb53e9cf1071dc8 changed from -1.0 to 0.0, refreshing cache 2024-11-25T03:41:17,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40325 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 0698446036b7ae35179ebc9b07274db5 changed from -1.0 to 0.0, refreshing cache 2024-11-25T03:41:17,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40325 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 706facb174cc9547b71f08aa672ddcee changed from -1.0 to 0.0, refreshing cache 2024-11-25T03:41:17,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40325 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region df71ad9490ccea77dfed73e590672db8 changed from -1.0 to 0.0, refreshing cache 2024-11-25T03:41:17,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40325 {}] rsgroup.RSGroupBasedLoadBalancer(138): Balancing RSGroup=default 2024-11-25T03:41:17,274 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40325 {}] rsgroup.RSGroupBasedLoadBalancer(151): Start Generate Balance plan for group: default 2024-11-25T03:41:17,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40325 {}] balancer.BaseLoadBalancer(619): Start Generate Balance plan for cluster. 2024-11-25T03:41:17,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40325 {}] balancer.BalancerClusterState(204): Hosts are {d73269e94d72=0} racks are {/default-rack=0} 2024-11-25T03:41:17,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40325 {}] balancer.BalancerClusterState(303): server 0 has 10 regions 2024-11-25T03:41:17,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40325 {}] balancer.BalancerClusterState(303): server 1 has 8 regions 2024-11-25T03:41:17,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40325 {}] balancer.BalancerClusterState(303): server 2 has 9 regions 2024-11-25T03:41:17,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40325 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-25T03:41:17,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40325 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-25T03:41:17,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40325 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-25T03:41:17,276 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40325 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-25T03:41:17,276 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40325 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-25T03:41:17,276 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40325 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-25T03:41:17,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40325 {}] balancer.BalancerClusterState(326): Number of tables=2, number of hosts=1, number of racks=1 2024-11-25T03:41:17,279 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40325 {}] balancer.StochasticLoadBalancer(403): Cluster wide - Calculating plan. may take up to 30000ms to complete. 2024-11-25T03:41:17,280 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40325 {}] balancer.StochasticLoadBalancer(515): Start StochasticLoadBalancer.balancer, initial weighted average imbalance=0.06931633435746164, functionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.06415002990995843, need balance); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.0); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); computedMaxSteps=64800 2024-11-25T03:41:17,745 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40325 {}] balancer.StochasticLoadBalancer(562): Finished computing new moving plan. Computation took 469 ms to try 64800 different iterations. Found a solution that moves 1 regions; Going from a computed imbalance of 0.06931633435746164 to a new imbalance of 0.01690158032826896. funtionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.0); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.037037037037037035, need balance); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); 2024-11-25T03:41:17,748 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40325 {}] master.HMaster(2167): Balancer plans size is 1, the balance interval is 300000 ms, and the max number regions in transition is 27 2024-11-25T03:41:17,748 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40325 {}] master.HMaster(2172): balance hri=1588230740, source=d73269e94d72,33145,1732506072477, destination=d73269e94d72,35981,1732506072374 2024-11-25T03:41:17,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40325 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, REOPEN/MOVE 2024-11-25T03:41:17,750 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, REOPEN/MOVE 2024-11-25T03:41:17,752 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=1588230740, regionState=CLOSING, regionLocation=d73269e94d72,33145,1732506072477 2024-11-25T03:41:17,754 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d73269e94d72,33145,1732506072477, state=CLOSING 2024-11-25T03:41:17,815 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40325 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=d73269e94d72,33145,1732506072477, table=hbase:meta, region=1588230740. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-25T03:41:17,819 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33145-0x1017014aa590002, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:17,819 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38397-0x1017014aa590003, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:17,819 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35981-0x1017014aa590001, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:17,819 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40325-0x1017014aa590000, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:17,819 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:17,819 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:17,819 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:17,819 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:17,820 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, REOPEN/MOVE 2024-11-25T03:41:17,820 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-25T03:41:17,820 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1588230740, server=d73269e94d72,33145,1732506072477}] 2024-11-25T03:41:17,981 INFO [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META, pid=58}] handler.UnassignRegionHandler(122): Close 1588230740 2024-11-25T03:41:17,981 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META, pid=58}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-25T03:41:17,982 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META, pid=58}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-25T03:41:17,982 INFO [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META, pid=58}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-25T03:41:17,982 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META, pid=58}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-25T03:41:17,982 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META, pid=58}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-25T03:41:17,983 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META, pid=58}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-25T03:41:17,989 INFO [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META, pid=58}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=28 KB heapSize=48.31 KB 2024-11-25T03:41:18,063 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META, pid=58}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/hbase/meta/1588230740/.tmp/info/d8db7666b3664908be7f9c3d0fe2748b is 138, key is test,bbb,1732506074951.c73ee0e3b504288792edda3ec9a26049./info:regioninfo/1732506076288/Put/seqid=0 2024-11-25T03:41:18,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41447 is added to blk_1073741864_1040 (size=34924) 2024-11-25T03:41:18,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33637 is added to blk_1073741864_1040 (size=34924) 2024-11-25T03:41:18,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741864_1040 (size=34924) 2024-11-25T03:41:18,478 INFO [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META, pid=58}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.86 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/hbase/meta/1588230740/.tmp/info/d8db7666b3664908be7f9c3d0fe2748b 2024-11-25T03:41:18,538 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META, pid=58}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/hbase/meta/1588230740/.tmp/ns/0e760b96fc624eed8ff5467f148c8465 is 43, key is default/ns:d/1732506074716/Put/seqid=0 2024-11-25T03:41:18,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33637 is added to blk_1073741865_1041 (size=5153) 2024-11-25T03:41:18,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41447 is added to blk_1073741865_1041 (size=5153) 2024-11-25T03:41:18,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741865_1041 (size=5153) 2024-11-25T03:41:18,551 INFO [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META, pid=58}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/hbase/meta/1588230740/.tmp/ns/0e760b96fc624eed8ff5467f148c8465 2024-11-25T03:41:18,576 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META, pid=58}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/hbase/meta/1588230740/.tmp/table/939abfc9f8c34cebba11b47fcfc9fcb1 is 40, key is test/table:state/1732506076403/Put/seqid=0 2024-11-25T03:41:18,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741866_1042 (size=5165) 2024-11-25T03:41:18,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33637 is added to blk_1073741866_1042 (size=5165) 2024-11-25T03:41:18,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41447 is added to blk_1073741866_1042 (size=5165) 2024-11-25T03:41:18,585 INFO [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META, pid=58}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=72 B at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/hbase/meta/1588230740/.tmp/table/939abfc9f8c34cebba11b47fcfc9fcb1 2024-11-25T03:41:18,599 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META, pid=58}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/hbase/meta/1588230740/.tmp/info/d8db7666b3664908be7f9c3d0fe2748b as hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/hbase/meta/1588230740/info/d8db7666b3664908be7f9c3d0fe2748b 2024-11-25T03:41:18,611 INFO [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META, pid=58}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/hbase/meta/1588230740/info/d8db7666b3664908be7f9c3d0fe2748b, entries=260, sequenceid=61, filesize=34.1 K 2024-11-25T03:41:18,614 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META, pid=58}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/hbase/meta/1588230740/.tmp/ns/0e760b96fc624eed8ff5467f148c8465 as hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/hbase/meta/1588230740/ns/0e760b96fc624eed8ff5467f148c8465 2024-11-25T03:41:18,625 INFO [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META, pid=58}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/hbase/meta/1588230740/ns/0e760b96fc624eed8ff5467f148c8465, entries=2, sequenceid=61, filesize=5.0 K 2024-11-25T03:41:18,627 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META, pid=58}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/hbase/meta/1588230740/.tmp/table/939abfc9f8c34cebba11b47fcfc9fcb1 as hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/hbase/meta/1588230740/table/939abfc9f8c34cebba11b47fcfc9fcb1 2024-11-25T03:41:18,639 INFO [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META, pid=58}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/hbase/meta/1588230740/table/939abfc9f8c34cebba11b47fcfc9fcb1, entries=2, sequenceid=61, filesize=5.0 K 2024-11-25T03:41:18,646 INFO [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META, pid=58}] regionserver.HRegion(3140): Finished flush of dataSize ~28 KB/28672, heapSize ~48.02 KB/49168, currentSize=0 B/0 for 1588230740 in 658ms, sequenceid=61, compaction requested=false 2024-11-25T03:41:18,648 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META, pid=58}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-25T03:41:18,660 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META, pid=58}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/hbase/meta/1588230740/recovered.edits/64.seqid, newMaxSeqId=64, maxSeqId=1 2024-11-25T03:41:18,663 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META, pid=58}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-25T03:41:18,664 INFO [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META, pid=58}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-25T03:41:18,664 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META, pid=58}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732506077982Running coprocessor pre-close hooks at 1732506077982Disabling compacts and flushes for region at 1732506077982Disabling writes for close at 1732506077982Obtaining lock to block concurrent updates at 1732506077989 (+7 ms)Preparing flush snapshotting stores in 1588230740 at 1732506077989Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=28672, getHeapSize=49408, getOffHeapSize=0, getCellsCount=264 at 1732506077998 (+9 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732506077999 (+1 ms)Flushing 1588230740/info: creating writer at 1732506078001 (+2 ms)Flushing 1588230740/info: appending metadata at 1732506078053 (+52 ms)Flushing 1588230740/info: closing flushed file at 1732506078056 (+3 ms)Flushing 1588230740/ns: creating writer at 1732506078521 (+465 ms)Flushing 1588230740/ns: appending metadata at 1732506078536 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1732506078536Flushing 1588230740/table: creating writer at 1732506078560 (+24 ms)Flushing 1588230740/table: appending metadata at 1732506078575 (+15 ms)Flushing 1588230740/table: closing flushed file at 1732506078575Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4618da0d: reopening flushed file at 1732506078597 (+22 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5907c3c7: reopening flushed file at 1732506078613 (+16 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b43bd95: reopening flushed file at 1732506078625 (+12 ms)Finished flush of dataSize ~28 KB/28672, heapSize ~48.02 KB/49168, currentSize=0 B/0 for 1588230740 in 658ms, sequenceid=61, compaction requested=false at 1732506078646 (+21 ms)Writing region close event to WAL at 1732506078655 (+9 ms)Running coprocessor post-close hooks at 1732506078661 (+6 ms)Closed at 1732506078664 (+3 ms) 2024-11-25T03:41:18,664 INFO [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META, pid=58}] regionserver.HRegionServer(3302): Adding 1588230740 move to d73269e94d72,35981,1732506072374 record at close sequenceid=61 2024-11-25T03:41:18,668 INFO [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META, pid=58}] handler.UnassignRegionHandler(157): Closed 1588230740 2024-11-25T03:41:18,669 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=1588230740, regionState=CLOSED 2024-11-25T03:41:18,670 WARN [PEWorker-1 {}] zookeeper.MetaTableLocator(168): Tried to set null ServerName in hbase:meta; skipping -- ServerName required 2024-11-25T03:41:18,670 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=58, ppid=57, state=RUNNABLE, hasLock=true; CloseRegionProcedure 1588230740, server=d73269e94d72,33145,1732506072477 2024-11-25T03:41:18,673 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-25T03:41:18,673 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; CloseRegionProcedure 1588230740, server=d73269e94d72,33145,1732506072477 in 850 msec 2024-11-25T03:41:18,674 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, REOPEN/MOVE; state=CLOSED, location=d73269e94d72,35981,1732506072374; forceNewPlan=false, retain=false 2024-11-25T03:41:18,824 INFO [d73269e94d72:40325 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-25T03:41:18,825 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=d73269e94d72,35981,1732506072374 2024-11-25T03:41:18,826 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d73269e94d72,35981,1732506072374, state=OPENING 2024-11-25T03:41:18,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33145-0x1017014aa590002, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:18,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40325-0x1017014aa590000, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:18,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35981-0x1017014aa590001, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:18,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38397-0x1017014aa590003, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:18,903 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, REOPEN/MOVE 2024-11-25T03:41:18,903 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:18,903 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:18,903 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:18,903 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:18,903 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=57, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=d73269e94d72,35981,1732506072374}] 2024-11-25T03:41:19,068 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=59}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-25T03:41:19,069 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=59}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T03:41:19,069 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=59}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-25T03:41:19,072 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=59}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d73269e94d72%2C35981%2C1732506072374.meta, suffix=.meta, logDir=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/WALs/d73269e94d72,35981,1732506072374, archiveDir=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/oldWALs, maxLogs=32 2024-11-25T03:41:19,092 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=59}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/WALs/d73269e94d72,35981,1732506072374/d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, exclude list is [], retry=0 2024-11-25T03:41:19,097 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33637,DS-88ab84a9-630b-478a-9aeb-f2157b7da108,DISK] 2024-11-25T03:41:19,097 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34311,DS-5d64405e-cf5f-4842-9c54-201a1e1e22d8,DISK] 2024-11-25T03:41:19,097 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41447,DS-43ff8d69-2bf0-4a76-b42b-90f4bc41dccf,DISK] 2024-11-25T03:41:19,105 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=59}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/WALs/d73269e94d72,35981,1732506072374/d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta 2024-11-25T03:41:19,105 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=59}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41877:41877),(127.0.0.1/127.0.0.1:35677:35677),(127.0.0.1/127.0.0.1:44951:44951)] 2024-11-25T03:41:19,105 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=59}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:41:19,106 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=59}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-25T03:41:19,106 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=59}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-25T03:41:19,106 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=59}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-25T03:41:19,106 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-25T03:41:19,106 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=59}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:41:19,106 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=59}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-25T03:41:19,106 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=59}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-25T03:41:19,108 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-25T03:41:19,109 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-25T03:41:19,109 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:41:19,119 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/hbase/meta/1588230740/info/d8db7666b3664908be7f9c3d0fe2748b 2024-11-25T03:41:19,119 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:41:19,119 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-25T03:41:19,121 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-25T03:41:19,121 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:41:19,131 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/hbase/meta/1588230740/ns/0e760b96fc624eed8ff5467f148c8465 2024-11-25T03:41:19,131 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:41:19,131 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-25T03:41:19,132 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-25T03:41:19,132 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:41:19,133 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:41:19,133 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-25T03:41:19,134 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-25T03:41:19,134 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:41:19,143 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/hbase/meta/1588230740/table/939abfc9f8c34cebba11b47fcfc9fcb1 2024-11-25T03:41:19,143 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:41:19,143 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=59}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-25T03:41:19,145 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/hbase/meta/1588230740 2024-11-25T03:41:19,146 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/hbase/meta/1588230740 2024-11-25T03:41:19,148 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=59}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-25T03:41:19,148 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=59}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-25T03:41:19,149 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=59}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-25T03:41:19,151 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=59}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-25T03:41:19,152 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=59}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=65; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67689350, jitterRate=0.00864991545677185}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-25T03:41:19,152 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=59}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-25T03:41:19,153 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=59}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732506079106Writing region info on filesystem at 1732506079106Initializing all the Stores at 1732506079107 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732506079107Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732506079108 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506079108Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732506079108Cleaning up temporary data from old regions at 1732506079148 (+40 ms)Running coprocessor post-open hooks at 1732506079152 (+4 ms)Region opened successfully at 1732506079153 (+1 ms) 2024-11-25T03:41:19,154 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=59}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=59, masterSystemTime=1732506079056 2024-11-25T03:41:19,157 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=59}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-25T03:41:19,157 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=59}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-25T03:41:19,158 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=65, regionLocation=d73269e94d72,35981,1732506072374 2024-11-25T03:41:19,159 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d73269e94d72,35981,1732506072374, state=OPEN 2024-11-25T03:41:19,166 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40325 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=d73269e94d72,35981,1732506072374, table=hbase:meta, region=1588230740. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-25T03:41:19,176 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35981-0x1017014aa590001, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:19,176 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38397-0x1017014aa590003, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:19,176 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33145-0x1017014aa590002, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:19,176 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40325-0x1017014aa590000, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:19,177 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:19,177 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:19,177 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:19,177 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:19,177 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=59, ppid=57, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=d73269e94d72,35981,1732506072374 2024-11-25T03:41:19,180 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=57 2024-11-25T03:41:19,180 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=57, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d73269e94d72,35981,1732506072374 in 274 msec 2024-11-25T03:41:19,183 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, REOPEN/MOVE in 1.4330 sec 2024-11-25T03:41:19,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40325 {}] master.HMaster(2203): Balancer is going into sleep until next period in 300000ms 2024-11-25T03:41:19,253 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [900,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T03:41:19,260 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40325 {}] master.MasterRpcServices(567): Client=jenkins//172.17.0.2 set balanceSwitch=false 2024-11-25T03:41:19,278 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.assignment.TestRollbackSCP#testFailAndRollback Thread=394, OpenFileDescriptor=651, MaxFileDescriptor=1048576, SystemLoadAverage=192, ProcessCount=11, AvailableMemoryMB=9265 2024-11-25T03:41:19,281 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table hbase:meta 2024-11-25T03:41:19,282 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=hbase:meta,,1 2024-11-25T03:41:19,282 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing d73269e94d72,35981,1732506072374 2024-11-25T03:41:19,282 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server d73269e94d72,35981,1732506072374: Simulated kill ***** 2024-11-25T03:41:19,282 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-25T03:41:19,284 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-25T03:41:19,286 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-25T03:41:19,286 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-25T03:41:19,286 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-25T03:41:19,287 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 376591872 }, "NonHeapMemoryUsage": { "committed": 156631040, "init": 7667712, "max": -1, "used": 152542456 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "d73269e94d72" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "d73269e94d72" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.Context": "regionserver", "tag.Hostname": "d73269e94d72" } ] } 2024-11-25T03:41:19,289 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40325 {}] master.MasterRpcServices(700): d73269e94d72,35981,1732506072374 reported a fatal error: ***** ABORTING region server d73269e94d72,35981,1732506072374: Simulated kill ***** 2024-11-25T03:41:19,291 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'd73269e94d72,35981,1732506072374' ***** 2024-11-25T03:41:19,291 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-11-25T03:41:19,291 INFO [RS:0;d73269e94d72:35981 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-25T03:41:19,292 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [15,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T03:41:19,292 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-25T03:41:19,292 INFO [RS:0;d73269e94d72:35981 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-25T03:41:19,292 INFO [RS:0;d73269e94d72:35981 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-25T03:41:19,292 INFO [RS:0;d73269e94d72:35981 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T03:41:19,292 INFO [RS:0;d73269e94d72:35981 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;d73269e94d72:35981. 2024-11-25T03:41:19,292 DEBUG [RS:0;d73269e94d72:35981 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T03:41:19,296 DEBUG [RS:0;d73269e94d72:35981 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:41:19,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41447 is added to blk_1073741867_1043 (size=436) 2024-11-25T03:41:19,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33637 is added to blk_1073741867_1043 (size=436) 2024-11-25T03:41:19,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741867_1043 (size=436) 2024-11-25T03:41:19,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741833_1009 (size=2232) 2024-11-25T03:41:19,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41447 is added to blk_1073741833_1009 (size=2232) 2024-11-25T03:41:19,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33637 is added to blk_1073741833_1009 (size=2232) 2024-11-25T03:41:19,308 DEBUG [RS:0;d73269e94d72:35981 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:41:19,308 INFO [RS:0;d73269e94d72:35981 {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T03:41:19,308 INFO [RS:0;d73269e94d72:35981 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35981 2024-11-25T03:41:19,324 INFO [RS:0;d73269e94d72:35981 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T03:41:19,324 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35981-0x1017014aa590001, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d73269e94d72,35981,1732506072374 2024-11-25T03:41:19,324 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40325-0x1017014aa590000, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T03:41:19,335 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d73269e94d72,35981,1732506072374] 2024-11-25T03:41:19,345 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/d73269e94d72,35981,1732506072374 already deleted, retry=false 2024-11-25T03:41:19,345 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of d73269e94d72,35981,1732506072374 on d73269e94d72,40325,1732506071573 2024-11-25T03:41:19,349 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true 2024-11-25T03:41:19,353 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=60 for d73269e94d72,35981,1732506072374 (carryingMeta=true) d73269e94d72,35981,1732506072374/CRASHED/regionCount=9/lock=java.util.concurrent.locks.ReentrantReadWriteLock@160df6ba[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-25T03:41:19,353 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=60, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true 2024-11-25T03:41:19,355 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=60, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true, isMeta: true 2024-11-25T03:41:19,357 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/WALs/d73269e94d72,35981,1732506072374-splitting 2024-11-25T03:41:19,359 INFO [PEWorker-1 {}] master.SplitWALManager(105): d73269e94d72,35981,1732506072374 WAL count=1, meta=true 2024-11-25T03:41:19,361 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta}] 2024-11-25T03:41:19,366 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=d73269e94d72,33145,1732506072477 2024-11-25T03:41:19,368 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477}] 2024-11-25T03:41:19,378 INFO [regionserver/d73269e94d72:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T03:41:19,393 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T03:41:19,435 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35981-0x1017014aa590001, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T03:41:19,435 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35981-0x1017014aa590001, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T03:41:19,436 INFO [RS:0;d73269e94d72:35981 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-25T03:41:19,436 INFO [RS:0;d73269e94d72:35981 {}] regionserver.HRegionServer(1031): Exiting; stopping=d73269e94d72,35981,1732506072374; zookeeper connection closed. 2024-11-25T03:41:19,436 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3c0b24c3 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3c0b24c3 2024-11-25T03:41:19,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33145 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=62 2024-11-25T03:41:19,551 INFO [RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(299): Splitting hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/WALs/d73269e94d72,35981,1732506072374-splitting/d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, size=436 (436bytes) 2024-11-25T03:41:19,552 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-25T03:41:19,552 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-25T03:41:19,552 INFO [RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/WALs/d73269e94d72,35981,1732506072374-splitting/d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta 2024-11-25T03:41:19,557 INFO [RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/WALs/d73269e94d72,35981,1732506072374-splitting/d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta after 3ms 2024-11-25T03:41:19,564 INFO [RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(310): Open hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/WALs/d73269e94d72,35981,1732506072374-splitting/d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta took 14ms 2024-11-25T03:41:19,572 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 61 store_sequence_id { family_name: "info" sequence_id: 61 } store_sequence_id { family_name: "ns" sequence_id: 61 } store_sequence_id { family_name: "rep_barrier" sequence_id: 61 } store_sequence_id { family_name: "table" sequence_id: 61 } 2024-11-25T03:41:19,572 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/WALs/d73269e94d72,35981,1732506072374-splitting/d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta so closing down 2024-11-25T03:41:19,572 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T03:41:19,573 INFO [RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T03:41:19,576 INFO [RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(425): Processed 1 edits across 0 Regions in 7 ms; skipped=1; WAL=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/WALs/d73269e94d72,35981,1732506072374-splitting/d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, size=436, length=436, corrupted=false, cancelled=false 2024-11-25T03:41:19,576 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(428): Completed split of hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/WALs/d73269e94d72,35981,1732506072374-splitting/d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, journal: Splitting hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/WALs/d73269e94d72,35981,1732506072374-splitting/d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, size=436 (436bytes) at 1732506079551Finishing writing output for hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/WALs/d73269e94d72,35981,1732506072374-splitting/d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta so closing down at 1732506079572 (+21 ms)3 split writer threads finished at 1732506079573 (+1 ms)Processed 1 edits across 0 Regions in 7 ms; skipped=1; WAL=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/WALs/d73269e94d72,35981,1732506072374-splitting/d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, size=436, length=436, corrupted=false, cancelled=false at 1732506079576 (+3 ms) 2024-11-25T03:41:19,576 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0-0 {event_type=RS_LOG_REPLAY, pid=62}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/WALs/d73269e94d72,35981,1732506072374-splitting/d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta 2024-11-25T03:41:19,577 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0-0 {event_type=RS_LOG_REPLAY, pid=62}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=62 2024-11-25T03:41:19,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40325 {}] master.HMaster(4169): Remote procedure done, pid=62 2024-11-25T03:41:19,586 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/WALs/d73269e94d72,35981,1732506072374-splitting/d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta to hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/oldWALs 2024-11-25T03:41:19,590 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-25T03:41:19,590 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 in 220 msec 2024-11-25T03:41:19,591 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=d73269e94d72,33145,1732506072477 2024-11-25T03:41:19,594 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-25T03:41:19,595 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 in 232 msec 2024-11-25T03:41:19,597 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/WALs/d73269e94d72,35981,1732506072374-splitting dir is empty, no logs to split. 2024-11-25T03:41:19,597 INFO [PEWorker-1 {}] master.SplitWALManager(105): d73269e94d72,35981,1732506072374 WAL count=0, meta=true 2024-11-25T03:41:19,597 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if d73269e94d72,35981,1732506072374 WAL splitting is done? wals=0, meta=true 2024-11-25T03:41:19,599 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-25T03:41:19,601 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T03:41:19,602 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-25T03:41:19,752 DEBUG [d73269e94d72:40325 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-25T03:41:19,752 DEBUG [d73269e94d72:40325 {}] balancer.BalancerClusterState(204): Hosts are {d73269e94d72=0} racks are {/default-rack=0} 2024-11-25T03:41:19,753 DEBUG [d73269e94d72:40325 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-25T03:41:19,753 DEBUG [d73269e94d72:40325 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-25T03:41:19,753 DEBUG [d73269e94d72:40325 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-25T03:41:19,753 DEBUG [d73269e94d72:40325 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-25T03:41:19,753 INFO [d73269e94d72:40325 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-25T03:41:19,753 INFO [d73269e94d72:40325 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-25T03:41:19,753 DEBUG [d73269e94d72:40325 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-25T03:41:19,753 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=63 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=d73269e94d72,33145,1732506072477 2024-11-25T03:41:19,754 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d73269e94d72,33145,1732506072477, state=OPENING 2024-11-25T03:41:19,797 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40325-0x1017014aa590000, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:19,797 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33145-0x1017014aa590002, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:19,797 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38397-0x1017014aa590003, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:19,798 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:19,798 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:19,798 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:19,797 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T03:41:19,798 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=d73269e94d72,33145,1732506072477}] 2024-11-25T03:41:19,959 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=64}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-25T03:41:19,959 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:41:19,960 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=64}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-25T03:41:19,961 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-25T03:41:19,961 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-25T03:41:19,961 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-25T03:41:19,961 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:41:19,962 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-25T03:41:19,962 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-25T03:41:19,965 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-25T03:41:19,966 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-25T03:41:19,966 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:41:19,975 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/hbase/meta/1588230740/info/d8db7666b3664908be7f9c3d0fe2748b 2024-11-25T03:41:19,975 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:41:19,975 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-25T03:41:19,976 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-25T03:41:19,976 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:41:19,986 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/hbase/meta/1588230740/ns/0e760b96fc624eed8ff5467f148c8465 2024-11-25T03:41:19,986 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:41:19,986 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-25T03:41:19,987 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-25T03:41:19,988 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:41:19,988 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:41:19,988 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-25T03:41:19,989 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-25T03:41:19,990 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:41:19,994 INFO [regionserver/d73269e94d72:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-25T03:41:19,994 INFO [regionserver/d73269e94d72:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-25T03:41:19,999 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/hbase/meta/1588230740/table/939abfc9f8c34cebba11b47fcfc9fcb1 2024-11-25T03:41:19,999 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:41:19,999 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-25T03:41:20,000 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/hbase/meta/1588230740 2024-11-25T03:41:20,001 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/hbase/meta/1588230740 2024-11-25T03:41:20,003 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-25T03:41:20,003 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-25T03:41:20,003 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-25T03:41:20,005 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-25T03:41:20,006 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=65; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65228480, jitterRate=-0.02801990509033203}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-25T03:41:20,006 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-25T03:41:20,006 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732506079962Writing region info on filesystem at 1732506079963 (+1 ms)Initializing all the Stores at 1732506079965 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732506079965Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732506079965Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506079965Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732506079965Cleaning up temporary data from old regions at 1732506080003 (+38 ms)Running coprocessor post-open hooks at 1732506080006 (+3 ms)Region opened successfully at 1732506080006 2024-11-25T03:41:20,007 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=64, masterSystemTime=1732506079950 2024-11-25T03:41:20,010 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-25T03:41:20,010 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=64}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-25T03:41:20,010 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=63 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=65, regionLocation=d73269e94d72,33145,1732506072477 2024-11-25T03:41:20,011 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d73269e94d72,33145,1732506072477, state=OPEN 2024-11-25T03:41:20,018 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38397-0x1017014aa590003, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:20,018 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33145-0x1017014aa590002, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:20,018 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40325-0x1017014aa590000, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:20,019 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=64, ppid=63, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=d73269e94d72,33145,1732506072477 2024-11-25T03:41:20,019 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:20,019 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:20,019 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:20,021 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-11-25T03:41:20,021 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d73269e94d72,33145,1732506072477 in 221 msec 2024-11-25T03:41:20,024 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=60 2024-11-25T03:41:20,024 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 422 msec 2024-11-25T03:41:20,024 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): d73269e94d72,35981,1732506072374 had 9 regions 2024-11-25T03:41:20,025 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=60, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true, isMeta: false 2024-11-25T03:41:20,027 INFO [PEWorker-1 {}] master.SplitWALManager(105): d73269e94d72,35981,1732506072374 WAL count=1, meta=false 2024-11-25T03:41:20,027 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=60, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123}] 2024-11-25T03:41:20,029 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=d73269e94d72,38397,1732506072528 2024-11-25T03:41:20,030 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528}] 2024-11-25T03:41:20,047 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T03:41:20,093 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test' 2024-11-25T03:41:20,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38397 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=66 2024-11-25T03:41:20,207 INFO [RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(299): Splitting hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/WALs/d73269e94d72,35981,1732506072374-splitting/d73269e94d72%2C35981%2C1732506072374.1732506074123, size=2.2 K (2232bytes) 2024-11-25T03:41:20,207 INFO [RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0-0 {event_type=RS_LOG_REPLAY, pid=66}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/WALs/d73269e94d72,35981,1732506072374-splitting/d73269e94d72%2C35981%2C1732506072374.1732506074123 2024-11-25T03:41:20,208 INFO [RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0-0 {event_type=RS_LOG_REPLAY, pid=66}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/WALs/d73269e94d72,35981,1732506072374-splitting/d73269e94d72%2C35981%2C1732506072374.1732506074123 after 1ms 2024-11-25T03:41:20,211 INFO [RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(310): Open hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/WALs/d73269e94d72,35981,1732506072374-splitting/d73269e94d72%2C35981%2C1732506072374.1732506074123 took 4ms 2024-11-25T03:41:20,214 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 4705fa4ae38907dd8b9783be2dff29da: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-25T03:41:20,216 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 857a6b6999e909a9b73e698c69291205: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-25T03:41:20,218 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 8b5ae4b02f22b24f40505bb660957004: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-25T03:41:20,219 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for c769ca025a015554efcc98619008be96: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-25T03:41:20,221 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 3b442507eaac3b5222a3c4f97ff80208: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-25T03:41:20,223 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 922ebf14d68aba40da3a2e99777c777b: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-25T03:41:20,224 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 2b224c7c31e6ff664ce67ac884f573d7: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-25T03:41:20,225 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for f5f64cb2042c701322ac00e312f65f16: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-25T03:41:20,226 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/WALs/d73269e94d72,35981,1732506072374-splitting/d73269e94d72%2C35981%2C1732506072374.1732506074123 so closing down 2024-11-25T03:41:20,226 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T03:41:20,226 INFO [RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T03:41:20,226 INFO [RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(425): Processed 8 edits across 0 Regions in 15 ms; skipped=8; WAL=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/WALs/d73269e94d72,35981,1732506072374-splitting/d73269e94d72%2C35981%2C1732506072374.1732506074123, size=2.2 K, length=2232, corrupted=false, cancelled=false 2024-11-25T03:41:20,226 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(428): Completed split of hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/WALs/d73269e94d72,35981,1732506072374-splitting/d73269e94d72%2C35981%2C1732506072374.1732506074123, journal: Splitting hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/WALs/d73269e94d72,35981,1732506072374-splitting/d73269e94d72%2C35981%2C1732506072374.1732506074123, size=2.2 K (2232bytes) at 1732506080207Finishing writing output for hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/WALs/d73269e94d72,35981,1732506072374-splitting/d73269e94d72%2C35981%2C1732506072374.1732506074123 so closing down at 1732506080226 (+19 ms)3 split writer threads finished at 1732506080226Processed 8 edits across 0 Regions in 15 ms; skipped=8; WAL=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/WALs/d73269e94d72,35981,1732506072374-splitting/d73269e94d72%2C35981%2C1732506072374.1732506074123, size=2.2 K, length=2232, corrupted=false, cancelled=false at 1732506080226 2024-11-25T03:41:20,226 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0-0 {event_type=RS_LOG_REPLAY, pid=66}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/WALs/d73269e94d72,35981,1732506072374-splitting/d73269e94d72%2C35981%2C1732506072374.1732506074123 2024-11-25T03:41:20,226 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0-0 {event_type=RS_LOG_REPLAY, pid=66}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=66 2024-11-25T03:41:20,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40325 {}] master.HMaster(4169): Remote procedure done, pid=66 2024-11-25T03:41:20,230 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/WALs/d73269e94d72,35981,1732506072374-splitting/d73269e94d72%2C35981%2C1732506072374.1732506074123 to hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/oldWALs 2024-11-25T03:41:20,232 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=66, resume processing ppid=65 2024-11-25T03:41:20,232 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 in 200 msec 2024-11-25T03:41:20,234 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=d73269e94d72,38397,1732506072528 2024-11-25T03:41:20,237 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=60 2024-11-25T03:41:20,237 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 in 207 msec 2024-11-25T03:41:20,239 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/WALs/d73269e94d72,35981,1732506072374-splitting dir is empty, no logs to split. 2024-11-25T03:41:20,240 INFO [PEWorker-1 {}] master.SplitWALManager(105): d73269e94d72,35981,1732506072374 WAL count=0, meta=false 2024-11-25T03:41:20,240 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if d73269e94d72,35981,1732506072374 WAL splitting is done? wals=0, meta=false 2024-11-25T03:41:20,242 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN}, {pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN}, {pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN}, {pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4705fa4ae38907dd8b9783be2dff29da, ASSIGN}, {pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c769ca025a015554efcc98619008be96, ASSIGN}, {pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8b5ae4b02f22b24f40505bb660957004, ASSIGN}, {pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f5f64cb2042c701322ac00e312f65f16, ASSIGN}, {pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2b224c7c31e6ff664ce67ac884f573d7, ASSIGN}] 2024-11-25T03:41:20,243 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2b224c7c31e6ff664ce67ac884f573d7, ASSIGN 2024-11-25T03:41:20,244 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=2b224c7c31e6ff664ce67ac884f573d7, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-25T03:41:20,245 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f5f64cb2042c701322ac00e312f65f16, ASSIGN 2024-11-25T03:41:20,246 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f5f64cb2042c701322ac00e312f65f16, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-25T03:41:20,247 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8b5ae4b02f22b24f40505bb660957004, ASSIGN 2024-11-25T03:41:20,248 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8b5ae4b02f22b24f40505bb660957004, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-25T03:41:20,248 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c769ca025a015554efcc98619008be96, ASSIGN 2024-11-25T03:41:20,249 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c769ca025a015554efcc98619008be96, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-25T03:41:20,250 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4705fa4ae38907dd8b9783be2dff29da, ASSIGN 2024-11-25T03:41:20,251 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4705fa4ae38907dd8b9783be2dff29da, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-25T03:41:20,252 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN 2024-11-25T03:41:20,252 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-25T03:41:20,253 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN 2024-11-25T03:41:20,254 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-25T03:41:20,254 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN 2024-11-25T03:41:20,255 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-25T03:41:20,395 DEBUG [d73269e94d72:40325 {}] balancer.BalancerClusterState(204): Hosts are {d73269e94d72=0} racks are {/default-rack=0} 2024-11-25T03:41:20,396 DEBUG [d73269e94d72:40325 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-25T03:41:20,397 DEBUG [d73269e94d72:40325 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-25T03:41:20,397 DEBUG [d73269e94d72:40325 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-25T03:41:20,397 DEBUG [d73269e94d72:40325 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-25T03:41:20,397 INFO [d73269e94d72:40325 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-25T03:41:20,397 INFO [d73269e94d72:40325 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-25T03:41:20,397 DEBUG [d73269e94d72:40325 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-25T03:41:20,399 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=74 updating hbase:meta row=2b224c7c31e6ff664ce67ac884f573d7, regionState=OPENING, regionLocation=d73269e94d72,33145,1732506072477 2024-11-25T03:41:20,400 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=922ebf14d68aba40da3a2e99777c777b, regionState=OPENING, regionLocation=d73269e94d72,33145,1732506072477 2024-11-25T03:41:20,401 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=73 updating hbase:meta row=f5f64cb2042c701322ac00e312f65f16, regionState=OPENING, regionLocation=d73269e94d72,33145,1732506072477 2024-11-25T03:41:20,401 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=3b442507eaac3b5222a3c4f97ff80208, regionState=OPENING, regionLocation=d73269e94d72,33145,1732506072477 2024-11-25T03:41:20,402 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=c769ca025a015554efcc98619008be96, regionState=OPENING, regionLocation=d73269e94d72,38397,1732506072528 2024-11-25T03:41:20,403 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=72 updating hbase:meta row=8b5ae4b02f22b24f40505bb660957004, regionState=OPENING, regionLocation=d73269e94d72,38397,1732506072528 2024-11-25T03:41:20,404 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=2b224c7c31e6ff664ce67ac884f573d7, ASSIGN because future has completed 2024-11-25T03:41:20,404 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b224c7c31e6ff664ce67ac884f573d7, server=d73269e94d72,33145,1732506072477}] 2024-11-25T03:41:20,408 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN because future has completed 2024-11-25T03:41:20,409 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f5f64cb2042c701322ac00e312f65f16, ASSIGN because future has completed 2024-11-25T03:41:20,410 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN because future has completed 2024-11-25T03:41:20,411 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477}] 2024-11-25T03:41:20,412 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c769ca025a015554efcc98619008be96, ASSIGN because future has completed 2024-11-25T03:41:20,413 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8b5ae4b02f22b24f40505bb660957004, ASSIGN because future has completed 2024-11-25T03:41:20,415 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8b5ae4b02f22b24f40505bb660957004, server=d73269e94d72,38397,1732506072528}] 2024-11-25T03:41:20,419 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure c769ca025a015554efcc98619008be96, server=d73269e94d72,38397,1732506072528}] 2024-11-25T03:41:20,423 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5f64cb2042c701322ac00e312f65f16, server=d73269e94d72,33145,1732506072477}] 2024-11-25T03:41:20,426 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 922ebf14d68aba40da3a2e99777c777b, server=d73269e94d72,33145,1732506072477}] 2024-11-25T03:41:20,429 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=4705fa4ae38907dd8b9783be2dff29da, regionState=OPENING, regionLocation=d73269e94d72,38397,1732506072528 2024-11-25T03:41:20,430 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=857a6b6999e909a9b73e698c69291205, regionState=OPENING, regionLocation=d73269e94d72,38397,1732506072528 2024-11-25T03:41:20,431 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4705fa4ae38907dd8b9783be2dff29da, ASSIGN because future has completed 2024-11-25T03:41:20,432 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4705fa4ae38907dd8b9783be2dff29da, server=d73269e94d72,38397,1732506072528}] 2024-11-25T03:41:20,432 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN because future has completed 2024-11-25T03:41:20,435 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 857a6b6999e909a9b73e698c69291205, server=d73269e94d72,38397,1732506072528}] 2024-11-25T03:41:20,565 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(132): Open test,,1732506074951.3b442507eaac3b5222a3c4f97ff80208. 2024-11-25T03:41:20,565 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7752): Opening region: {ENCODED => 3b442507eaac3b5222a3c4f97ff80208, NAME => 'test,,1732506074951.3b442507eaac3b5222a3c4f97ff80208.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-25T03:41:20,566 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 3b442507eaac3b5222a3c4f97ff80208 2024-11-25T03:41:20,566 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(898): Instantiated test,,1732506074951.3b442507eaac3b5222a3c4f97ff80208.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:41:20,566 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7794): checking encryption for 3b442507eaac3b5222a3c4f97ff80208 2024-11-25T03:41:20,566 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7797): checking classloading for 3b442507eaac3b5222a3c4f97ff80208 2024-11-25T03:41:20,569 INFO [StoreOpener-3b442507eaac3b5222a3c4f97ff80208-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 3b442507eaac3b5222a3c4f97ff80208 2024-11-25T03:41:20,571 INFO [StoreOpener-3b442507eaac3b5222a3c4f97ff80208-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3b442507eaac3b5222a3c4f97ff80208 columnFamilyName family 2024-11-25T03:41:20,571 DEBUG [StoreOpener-3b442507eaac3b5222a3c4f97ff80208-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:41:20,572 INFO [StoreOpener-3b442507eaac3b5222a3c4f97ff80208-1 {}] regionserver.HStore(327): Store=3b442507eaac3b5222a3c4f97ff80208/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:41:20,572 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1038): replaying wal for 3b442507eaac3b5222a3c4f97ff80208 2024-11-25T03:41:20,573 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] handler.AssignRegionHandler(132): Open test,ggg,1732506074951.857a6b6999e909a9b73e698c69291205. 2024-11-25T03:41:20,573 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7752): Opening region: {ENCODED => 857a6b6999e909a9b73e698c69291205, NAME => 'test,ggg,1732506074951.857a6b6999e909a9b73e698c69291205.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-11-25T03:41:20,573 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 857a6b6999e909a9b73e698c69291205 2024-11-25T03:41:20,573 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(898): Instantiated test,ggg,1732506074951.857a6b6999e909a9b73e698c69291205.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:41:20,573 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7794): checking encryption for 857a6b6999e909a9b73e698c69291205 2024-11-25T03:41:20,573 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7797): checking classloading for 857a6b6999e909a9b73e698c69291205 2024-11-25T03:41:20,573 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/3b442507eaac3b5222a3c4f97ff80208 2024-11-25T03:41:20,575 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/3b442507eaac3b5222a3c4f97ff80208 2024-11-25T03:41:20,575 INFO [StoreOpener-857a6b6999e909a9b73e698c69291205-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 857a6b6999e909a9b73e698c69291205 2024-11-25T03:41:20,575 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1048): stopping wal replay for 3b442507eaac3b5222a3c4f97ff80208 2024-11-25T03:41:20,575 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1060): Cleaning up temporary data for 3b442507eaac3b5222a3c4f97ff80208 2024-11-25T03:41:20,576 INFO [StoreOpener-857a6b6999e909a9b73e698c69291205-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 857a6b6999e909a9b73e698c69291205 columnFamilyName family 2024-11-25T03:41:20,576 DEBUG [StoreOpener-857a6b6999e909a9b73e698c69291205-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:41:20,576 INFO [StoreOpener-857a6b6999e909a9b73e698c69291205-1 {}] regionserver.HStore(327): Store=857a6b6999e909a9b73e698c69291205/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:41:20,577 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1038): replaying wal for 857a6b6999e909a9b73e698c69291205 2024-11-25T03:41:20,577 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1093): writing seq id for 3b442507eaac3b5222a3c4f97ff80208 2024-11-25T03:41:20,577 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/857a6b6999e909a9b73e698c69291205 2024-11-25T03:41:20,578 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1114): Opened 3b442507eaac3b5222a3c4f97ff80208; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70559384, jitterRate=0.051416754722595215}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:41:20,578 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3b442507eaac3b5222a3c4f97ff80208 2024-11-25T03:41:20,578 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1006): Region open journal for 3b442507eaac3b5222a3c4f97ff80208: Running coprocessor pre-open hook at 1732506080567Writing region info on filesystem at 1732506080567Initializing all the Stores at 1732506080569 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506080569Cleaning up temporary data from old regions at 1732506080575 (+6 ms)Running coprocessor post-open hooks at 1732506080578 (+3 ms)Region opened successfully at 1732506080578 2024-11-25T03:41:20,579 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/857a6b6999e909a9b73e698c69291205 2024-11-25T03:41:20,579 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1048): stopping wal replay for 857a6b6999e909a9b73e698c69291205 2024-11-25T03:41:20,579 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1732506074951.3b442507eaac3b5222a3c4f97ff80208., pid=76, masterSystemTime=1732506080559 2024-11-25T03:41:20,579 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1060): Cleaning up temporary data for 857a6b6999e909a9b73e698c69291205 2024-11-25T03:41:20,581 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1093): writing seq id for 857a6b6999e909a9b73e698c69291205 2024-11-25T03:41:20,582 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2266): Finished post open deploy task for test,,1732506074951.3b442507eaac3b5222a3c4f97ff80208. 2024-11-25T03:41:20,582 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(153): Opened test,,1732506074951.3b442507eaac3b5222a3c4f97ff80208. 2024-11-25T03:41:20,582 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(132): Open test,www,1732506074951.2b224c7c31e6ff664ce67ac884f573d7. 2024-11-25T03:41:20,582 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7752): Opening region: {ENCODED => 2b224c7c31e6ff664ce67ac884f573d7, NAME => 'test,www,1732506074951.2b224c7c31e6ff664ce67ac884f573d7.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-11-25T03:41:20,582 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 2b224c7c31e6ff664ce67ac884f573d7 2024-11-25T03:41:20,582 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(898): Instantiated test,www,1732506074951.2b224c7c31e6ff664ce67ac884f573d7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:41:20,582 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7794): checking encryption for 2b224c7c31e6ff664ce67ac884f573d7 2024-11-25T03:41:20,583 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7797): checking classloading for 2b224c7c31e6ff664ce67ac884f573d7 2024-11-25T03:41:20,582 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1114): Opened 857a6b6999e909a9b73e698c69291205; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66466691, jitterRate=-0.009569123387336731}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:41:20,583 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 857a6b6999e909a9b73e698c69291205 2024-11-25T03:41:20,583 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1006): Region open journal for 857a6b6999e909a9b73e698c69291205: Running coprocessor pre-open hook at 1732506080574Writing region info on filesystem at 1732506080574Initializing all the Stores at 1732506080574Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506080574Cleaning up temporary data from old regions at 1732506080579 (+5 ms)Running coprocessor post-open hooks at 1732506080583 (+4 ms)Region opened successfully at 1732506080583 2024-11-25T03:41:20,584 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ggg,1732506074951.857a6b6999e909a9b73e698c69291205., pid=82, masterSystemTime=1732506080567 2024-11-25T03:41:20,584 INFO [StoreOpener-2b224c7c31e6ff664ce67ac884f573d7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 2b224c7c31e6ff664ce67ac884f573d7 2024-11-25T03:41:20,585 WARN [PEWorker-1 {}] procedure2.ProcedureTestingUtility(201): Set Kill before store update in rollback to: true 2024-11-25T03:41:20,585 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477 2024-11-25T03:41:20,585 INFO [StoreOpener-2b224c7c31e6ff664ce67ac884f573d7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2b224c7c31e6ff664ce67ac884f573d7 columnFamilyName family 2024-11-25T03:41:20,585 DEBUG [StoreOpener-2b224c7c31e6ff664ce67ac884f573d7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:41:20,586 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ggg,1732506074951.857a6b6999e909a9b73e698c69291205. 2024-11-25T03:41:20,586 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] handler.AssignRegionHandler(153): Opened test,ggg,1732506074951.857a6b6999e909a9b73e698c69291205. 2024-11-25T03:41:20,586 INFO [StoreOpener-2b224c7c31e6ff664ce67ac884f573d7-1 {}] regionserver.HStore(327): Store=2b224c7c31e6ff664ce67ac884f573d7/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:41:20,586 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(132): Open test,nnn,1732506074951.4705fa4ae38907dd8b9783be2dff29da. 2024-11-25T03:41:20,586 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1038): replaying wal for 2b224c7c31e6ff664ce67ac884f573d7 2024-11-25T03:41:20,586 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7752): Opening region: {ENCODED => 4705fa4ae38907dd8b9783be2dff29da, NAME => 'test,nnn,1732506074951.4705fa4ae38907dd8b9783be2dff29da.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-11-25T03:41:20,587 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4705fa4ae38907dd8b9783be2dff29da 2024-11-25T03:41:20,587 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(898): Instantiated test,nnn,1732506074951.4705fa4ae38907dd8b9783be2dff29da.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:41:20,586 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1844): CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477 java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T03:41:20,587 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/2b224c7c31e6ff664ce67ac884f573d7 2024-11-25T03:41:20,587 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7794): checking encryption for 4705fa4ae38907dd8b9783be2dff29da 2024-11-25T03:41:20,587 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7797): checking classloading for 4705fa4ae38907dd8b9783be2dff29da 2024-11-25T03:41:20,589 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/2b224c7c31e6ff664ce67ac884f573d7 2024-11-25T03:41:20,589 INFO [StoreOpener-4705fa4ae38907dd8b9783be2dff29da-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4705fa4ae38907dd8b9783be2dff29da 2024-11-25T03:41:20,589 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1048): stopping wal replay for 2b224c7c31e6ff664ce67ac884f573d7 2024-11-25T03:41:20,589 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1060): Cleaning up temporary data for 2b224c7c31e6ff664ce67ac884f573d7 2024-11-25T03:41:20,590 INFO [StoreOpener-4705fa4ae38907dd8b9783be2dff29da-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4705fa4ae38907dd8b9783be2dff29da columnFamilyName family 2024-11-25T03:41:20,590 DEBUG [StoreOpener-4705fa4ae38907dd8b9783be2dff29da-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:41:20,591 INFO [StoreOpener-4705fa4ae38907dd8b9783be2dff29da-1 {}] regionserver.HStore(327): Store=4705fa4ae38907dd8b9783be2dff29da/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:41:20,591 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1038): replaying wal for 4705fa4ae38907dd8b9783be2dff29da 2024-11-25T03:41:20,591 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): writing seq id for 2b224c7c31e6ff664ce67ac884f573d7 2024-11-25T03:41:20,592 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/4705fa4ae38907dd8b9783be2dff29da 2024-11-25T03:41:20,593 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1114): Opened 2b224c7c31e6ff664ce67ac884f573d7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63827216, jitterRate=-0.04890036582946777}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:41:20,593 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2b224c7c31e6ff664ce67ac884f573d7 2024-11-25T03:41:20,593 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/4705fa4ae38907dd8b9783be2dff29da 2024-11-25T03:41:20,593 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1006): Region open journal for 2b224c7c31e6ff664ce67ac884f573d7: Running coprocessor pre-open hook at 1732506080583Writing region info on filesystem at 1732506080583Initializing all the Stores at 1732506080584 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506080584Cleaning up temporary data from old regions at 1732506080589 (+5 ms)Running coprocessor post-open hooks at 1732506080593 (+4 ms)Region opened successfully at 1732506080593 2024-11-25T03:41:20,593 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1048): stopping wal replay for 4705fa4ae38907dd8b9783be2dff29da 2024-11-25T03:41:20,593 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1060): Cleaning up temporary data for 4705fa4ae38907dd8b9783be2dff29da 2024-11-25T03:41:20,594 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2236): Post open deploy tasks for test,www,1732506074951.2b224c7c31e6ff664ce67ac884f573d7., pid=75, masterSystemTime=1732506080559 2024-11-25T03:41:20,595 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1093): writing seq id for 4705fa4ae38907dd8b9783be2dff29da 2024-11-25T03:41:20,596 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2266): Finished post open deploy task for test,www,1732506074951.2b224c7c31e6ff664ce67ac884f573d7. 2024-11-25T03:41:20,596 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(153): Opened test,www,1732506074951.2b224c7c31e6ff664ce67ac884f573d7. 2024-11-25T03:41:20,596 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(132): Open test,qqq,1732506074951.922ebf14d68aba40da3a2e99777c777b. 2024-11-25T03:41:20,596 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1114): Opened 4705fa4ae38907dd8b9783be2dff29da; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63964951, jitterRate=-0.046847954392433167}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:41:20,597 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7752): Opening region: {ENCODED => 922ebf14d68aba40da3a2e99777c777b, NAME => 'test,qqq,1732506074951.922ebf14d68aba40da3a2e99777c777b.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-11-25T03:41:20,597 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4705fa4ae38907dd8b9783be2dff29da 2024-11-25T03:41:20,597 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1006): Region open journal for 4705fa4ae38907dd8b9783be2dff29da: Running coprocessor pre-open hook at 1732506080587Writing region info on filesystem at 1732506080587Initializing all the Stores at 1732506080588 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506080588Cleaning up temporary data from old regions at 1732506080593 (+5 ms)Running coprocessor post-open hooks at 1732506080597 (+4 ms)Region opened successfully at 1732506080597 2024-11-25T03:41:20,597 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 922ebf14d68aba40da3a2e99777c777b 2024-11-25T03:41:20,597 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(898): Instantiated test,qqq,1732506074951.922ebf14d68aba40da3a2e99777c777b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:41:20,597 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7794): checking encryption for 922ebf14d68aba40da3a2e99777c777b 2024-11-25T03:41:20,597 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7797): checking classloading for 922ebf14d68aba40da3a2e99777c777b 2024-11-25T03:41:20,598 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2236): Post open deploy tasks for test,nnn,1732506074951.4705fa4ae38907dd8b9783be2dff29da., pid=81, masterSystemTime=1732506080567 2024-11-25T03:41:20,599 INFO [StoreOpener-922ebf14d68aba40da3a2e99777c777b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 922ebf14d68aba40da3a2e99777c777b 2024-11-25T03:41:20,598 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=true, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1845) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 4 more 2024-11-25T03:41:20,600 INFO [StoreOpener-922ebf14d68aba40da3a2e99777c777b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 922ebf14d68aba40da3a2e99777c777b columnFamilyName family 2024-11-25T03:41:20,600 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2266): Finished post open deploy task for test,nnn,1732506074951.4705fa4ae38907dd8b9783be2dff29da. 2024-11-25T03:41:20,600 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(153): Opened test,nnn,1732506074951.4705fa4ae38907dd8b9783be2dff29da. 2024-11-25T03:41:20,600 DEBUG [StoreOpener-922ebf14d68aba40da3a2e99777c777b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:41:20,600 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(132): Open test,zzz,1732506074951.c769ca025a015554efcc98619008be96. 2024-11-25T03:41:20,600 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7752): Opening region: {ENCODED => c769ca025a015554efcc98619008be96, NAME => 'test,zzz,1732506074951.c769ca025a015554efcc98619008be96.', STARTKEY => 'zzz', ENDKEY => ''} 2024-11-25T03:41:20,600 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-25T03:41:20,600 INFO [StoreOpener-922ebf14d68aba40da3a2e99777c777b-1 {}] regionserver.HStore(327): Store=922ebf14d68aba40da3a2e99777c777b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:41:20,601 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=82, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 857a6b6999e909a9b73e698c69291205, server=d73269e94d72,38397,1732506072528 2024-11-25T03:41:20,601 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c769ca025a015554efcc98619008be96 2024-11-25T03:41:20,601 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T03:41:20,601 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1038): replaying wal for 922ebf14d68aba40da3a2e99777c777b 2024-11-25T03:41:20,601 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(898): Instantiated test,zzz,1732506074951.c769ca025a015554efcc98619008be96.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:41:20,601 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7794): checking encryption for c769ca025a015554efcc98619008be96 2024-11-25T03:41:20,601 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7797): checking classloading for c769ca025a015554efcc98619008be96 2024-11-25T03:41:20,601 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=82, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 857a6b6999e909a9b73e698c69291205, server=d73269e94d72,38397,1732506072528 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T03:41:20,602 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/922ebf14d68aba40da3a2e99777c777b 2024-11-25T03:41:20,602 INFO [StoreOpener-c769ca025a015554efcc98619008be96-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c769ca025a015554efcc98619008be96 2024-11-25T03:41:20,603 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/922ebf14d68aba40da3a2e99777c777b 2024-11-25T03:41:20,603 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1048): stopping wal replay for 922ebf14d68aba40da3a2e99777c777b 2024-11-25T03:41:20,603 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1060): Cleaning up temporary data for 922ebf14d68aba40da3a2e99777c777b 2024-11-25T03:41:20,603 INFO [StoreOpener-c769ca025a015554efcc98619008be96-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c769ca025a015554efcc98619008be96 columnFamilyName family 2024-11-25T03:41:20,603 DEBUG [StoreOpener-c769ca025a015554efcc98619008be96-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:41:20,604 INFO [StoreOpener-c769ca025a015554efcc98619008be96-1 {}] regionserver.HStore(327): Store=c769ca025a015554efcc98619008be96/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:41:20,604 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1038): replaying wal for c769ca025a015554efcc98619008be96 2024-11-25T03:41:20,605 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/c769ca025a015554efcc98619008be96 2024-11-25T03:41:20,605 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1093): writing seq id for 922ebf14d68aba40da3a2e99777c777b 2024-11-25T03:41:20,606 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/c769ca025a015554efcc98619008be96 2024-11-25T03:41:20,606 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1114): Opened 922ebf14d68aba40da3a2e99777c777b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67338440, jitterRate=0.0034209489822387695}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:41:20,606 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1048): stopping wal replay for c769ca025a015554efcc98619008be96 2024-11-25T03:41:20,606 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 922ebf14d68aba40da3a2e99777c777b 2024-11-25T03:41:20,606 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1060): Cleaning up temporary data for c769ca025a015554efcc98619008be96 2024-11-25T03:41:20,606 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1006): Region open journal for 922ebf14d68aba40da3a2e99777c777b: Running coprocessor pre-open hook at 1732506080597Writing region info on filesystem at 1732506080597Initializing all the Stores at 1732506080598 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506080598Cleaning up temporary data from old regions at 1732506080603 (+5 ms)Running coprocessor post-open hooks at 1732506080606 (+3 ms)Region opened successfully at 1732506080606 2024-11-25T03:41:20,607 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2236): Post open deploy tasks for test,qqq,1732506074951.922ebf14d68aba40da3a2e99777c777b., pid=80, masterSystemTime=1732506080559 2024-11-25T03:41:20,608 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1093): writing seq id for c769ca025a015554efcc98619008be96 2024-11-25T03:41:20,609 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2266): Finished post open deploy task for test,qqq,1732506074951.922ebf14d68aba40da3a2e99777c777b. 2024-11-25T03:41:20,610 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(153): Opened test,qqq,1732506074951.922ebf14d68aba40da3a2e99777c777b. 2024-11-25T03:41:20,610 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1114): Opened c769ca025a015554efcc98619008be96; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60793428, jitterRate=-0.09410732984542847}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:41:20,610 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c769ca025a015554efcc98619008be96 2024-11-25T03:41:20,610 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(132): Open test,jjj,1732506074951.f5f64cb2042c701322ac00e312f65f16. 2024-11-25T03:41:20,610 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1006): Region open journal for c769ca025a015554efcc98619008be96: Running coprocessor pre-open hook at 1732506080601Writing region info on filesystem at 1732506080601Initializing all the Stores at 1732506080602 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506080602Cleaning up temporary data from old regions at 1732506080606 (+4 ms)Running coprocessor post-open hooks at 1732506080610 (+4 ms)Region opened successfully at 1732506080610 2024-11-25T03:41:20,610 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7752): Opening region: {ENCODED => f5f64cb2042c701322ac00e312f65f16, NAME => 'test,jjj,1732506074951.f5f64cb2042c701322ac00e312f65f16.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-25T03:41:20,610 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f5f64cb2042c701322ac00e312f65f16 2024-11-25T03:41:20,610 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(898): Instantiated test,jjj,1732506074951.f5f64cb2042c701322ac00e312f65f16.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:41:20,610 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7794): checking encryption for f5f64cb2042c701322ac00e312f65f16 2024-11-25T03:41:20,610 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7797): checking classloading for f5f64cb2042c701322ac00e312f65f16 2024-11-25T03:41:20,611 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2236): Post open deploy tasks for test,zzz,1732506074951.c769ca025a015554efcc98619008be96., pid=78, masterSystemTime=1732506080567 2024-11-25T03:41:20,613 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2266): Finished post open deploy task for test,zzz,1732506074951.c769ca025a015554efcc98619008be96. 2024-11-25T03:41:20,613 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(153): Opened test,zzz,1732506074951.c769ca025a015554efcc98619008be96. 2024-11-25T03:41:20,613 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(132): Open test,ttt,1732506074951.8b5ae4b02f22b24f40505bb660957004. 2024-11-25T03:41:20,614 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7752): Opening region: {ENCODED => 8b5ae4b02f22b24f40505bb660957004, NAME => 'test,ttt,1732506074951.8b5ae4b02f22b24f40505bb660957004.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-25T03:41:20,614 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8b5ae4b02f22b24f40505bb660957004 2024-11-25T03:41:20,614 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(898): Instantiated test,ttt,1732506074951.8b5ae4b02f22b24f40505bb660957004.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:41:20,614 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7794): checking encryption for 8b5ae4b02f22b24f40505bb660957004 2024-11-25T03:41:20,614 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7797): checking classloading for 8b5ae4b02f22b24f40505bb660957004 2024-11-25T03:41:20,615 INFO [StoreOpener-f5f64cb2042c701322ac00e312f65f16-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f5f64cb2042c701322ac00e312f65f16 2024-11-25T03:41:20,616 INFO [StoreOpener-f5f64cb2042c701322ac00e312f65f16-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f5f64cb2042c701322ac00e312f65f16 columnFamilyName family 2024-11-25T03:41:20,616 DEBUG [StoreOpener-f5f64cb2042c701322ac00e312f65f16-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:41:20,617 INFO [StoreOpener-8b5ae4b02f22b24f40505bb660957004-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8b5ae4b02f22b24f40505bb660957004 2024-11-25T03:41:20,617 INFO [StoreOpener-f5f64cb2042c701322ac00e312f65f16-1 {}] regionserver.HStore(327): Store=f5f64cb2042c701322ac00e312f65f16/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:41:20,618 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1038): replaying wal for f5f64cb2042c701322ac00e312f65f16 2024-11-25T03:41:20,618 INFO [StoreOpener-8b5ae4b02f22b24f40505bb660957004-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8b5ae4b02f22b24f40505bb660957004 columnFamilyName family 2024-11-25T03:41:20,618 DEBUG [StoreOpener-8b5ae4b02f22b24f40505bb660957004-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:41:20,618 INFO [StoreOpener-8b5ae4b02f22b24f40505bb660957004-1 {}] regionserver.HStore(327): Store=8b5ae4b02f22b24f40505bb660957004/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:41:20,619 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1038): replaying wal for 8b5ae4b02f22b24f40505bb660957004 2024-11-25T03:41:20,619 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/8b5ae4b02f22b24f40505bb660957004 2024-11-25T03:41:20,620 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/f5f64cb2042c701322ac00e312f65f16 2024-11-25T03:41:20,621 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/8b5ae4b02f22b24f40505bb660957004 2024-11-25T03:41:20,621 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/f5f64cb2042c701322ac00e312f65f16 2024-11-25T03:41:20,621 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1048): stopping wal replay for 8b5ae4b02f22b24f40505bb660957004 2024-11-25T03:41:20,621 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1060): Cleaning up temporary data for 8b5ae4b02f22b24f40505bb660957004 2024-11-25T03:41:20,622 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1048): stopping wal replay for f5f64cb2042c701322ac00e312f65f16 2024-11-25T03:41:20,622 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1060): Cleaning up temporary data for f5f64cb2042c701322ac00e312f65f16 2024-11-25T03:41:20,624 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1093): writing seq id for f5f64cb2042c701322ac00e312f65f16 2024-11-25T03:41:20,624 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1093): writing seq id for 8b5ae4b02f22b24f40505bb660957004 2024-11-25T03:41:20,625 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1114): Opened f5f64cb2042c701322ac00e312f65f16; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64042431, jitterRate=-0.04569341242313385}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:41:20,625 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1114): Opened 8b5ae4b02f22b24f40505bb660957004; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61508172, jitterRate=-0.08345681428909302}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:41:20,625 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f5f64cb2042c701322ac00e312f65f16 2024-11-25T03:41:20,625 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8b5ae4b02f22b24f40505bb660957004 2024-11-25T03:41:20,625 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1006): Region open journal for f5f64cb2042c701322ac00e312f65f16: Running coprocessor pre-open hook at 1732506080611Writing region info on filesystem at 1732506080611Initializing all the Stores at 1732506080612 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506080612Cleaning up temporary data from old regions at 1732506080622 (+10 ms)Running coprocessor post-open hooks at 1732506080625 (+3 ms)Region opened successfully at 1732506080625 2024-11-25T03:41:20,625 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1006): Region open journal for 8b5ae4b02f22b24f40505bb660957004: Running coprocessor pre-open hook at 1732506080614Writing region info on filesystem at 1732506080614Initializing all the Stores at 1732506080615 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506080616 (+1 ms)Cleaning up temporary data from old regions at 1732506080621 (+5 ms)Running coprocessor post-open hooks at 1732506080625 (+4 ms)Region opened successfully at 1732506080625 2024-11-25T03:41:20,626 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2236): Post open deploy tasks for test,jjj,1732506074951.f5f64cb2042c701322ac00e312f65f16., pid=79, masterSystemTime=1732506080559 2024-11-25T03:41:20,626 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1732506074951.8b5ae4b02f22b24f40505bb660957004., pid=77, masterSystemTime=1732506080567 2024-11-25T03:41:20,628 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2266): Finished post open deploy task for test,jjj,1732506074951.f5f64cb2042c701322ac00e312f65f16. 2024-11-25T03:41:20,629 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(153): Opened test,jjj,1732506074951.f5f64cb2042c701322ac00e312f65f16. 2024-11-25T03:41:20,630 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1732506074951.8b5ae4b02f22b24f40505bb660957004. 2024-11-25T03:41:20,630 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(153): Opened test,ttt,1732506074951.8b5ae4b02f22b24f40505bb660957004. 2024-11-25T03:41:20,696 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-25T03:41:20,696 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T03:41:20,697 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-25T03:41:20,697 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-25T03:41:20,698 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-25T03:41:20,698 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-25T03:41:20,698 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-25T03:41:20,698 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-25T03:41:20,699 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-25T03:41:20,704 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-25T03:41:20,705 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-25T03:41:20,705 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-25T03:41:20,705 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:20,705 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=82, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-25T03:41:20,705 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:20,705 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-25T03:41:20,705 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:20,705 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-25T03:41:20,705 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:20,706 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=81, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-25T03:41:20,706 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:20,706 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-25T03:41:20,706 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:20,706 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-25T03:41:20,706 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:20,706 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-25T03:41:20,706 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:20,706 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-25T03:41:20,706 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-25T03:41:20,706 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-25T03:41:20,706 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:20,706 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-25T03:41:20,707 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-25T03:41:20,708 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-25T03:41:20,708 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, REOPEN/MOVE 2024-11-25T03:41:20,712 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true 2024-11-25T03:41:20,712 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 2024-11-25T03:41:20,712 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 2024-11-25T03:41:20,712 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T03:41:20,712 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d73269e94d72,33145,1732506072477 2024-11-25T03:41:20,712 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 2024-11-25T03:41:20,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 2024-11-25T03:41:20,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN 2024-11-25T03:41:20,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN 2024-11-25T03:41:20,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN 2024-11-25T03:41:20,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4705fa4ae38907dd8b9783be2dff29da, ASSIGN 2024-11-25T03:41:20,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c769ca025a015554efcc98619008be96, ASSIGN 2024-11-25T03:41:20,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b5ae4b02f22b24f40505bb660957004, ASSIGN 2024-11-25T03:41:20,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f64cb2042c701322ac00e312f65f16, ASSIGN 2024-11-25T03:41:20,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b224c7c31e6ff664ce67ac884f573d7, ASSIGN 2024-11-25T03:41:20,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b224c7c31e6ff664ce67ac884f573d7, server=d73269e94d72,33145,1732506072477 2024-11-25T03:41:20,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477 2024-11-25T03:41:20,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8b5ae4b02f22b24f40505bb660957004, server=d73269e94d72,38397,1732506072528 2024-11-25T03:41:20,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure c769ca025a015554efcc98619008be96, server=d73269e94d72,38397,1732506072528 2024-11-25T03:41:20,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5f64cb2042c701322ac00e312f65f16, server=d73269e94d72,33145,1732506072477 2024-11-25T03:41:20,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 922ebf14d68aba40da3a2e99777c777b, server=d73269e94d72,33145,1732506072477 2024-11-25T03:41:20,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=81, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4705fa4ae38907dd8b9783be2dff29da, server=d73269e94d72,38397,1732506072528 2024-11-25T03:41:20,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=82, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 857a6b6999e909a9b73e698c69291205, server=d73269e94d72,38397,1732506072528 2024-11-25T03:41:20,713 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:20,713 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:20,713 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:20,713 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:20,713 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:20,713 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:20,713 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:20,713 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:20,714 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN 2024-11-25T03:41:20,714 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:20,714 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN 2024-11-25T03:41:20,714 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:20,714 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN 2024-11-25T03:41:20,714 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4705fa4ae38907dd8b9783be2dff29da, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:20,714 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4705fa4ae38907dd8b9783be2dff29da, ASSIGN 2024-11-25T03:41:20,714 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c769ca025a015554efcc98619008be96, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:20,714 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c769ca025a015554efcc98619008be96, ASSIGN 2024-11-25T03:41:20,714 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b5ae4b02f22b24f40505bb660957004, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:20,714 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b5ae4b02f22b24f40505bb660957004, ASSIGN 2024-11-25T03:41:20,714 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f64cb2042c701322ac00e312f65f16, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:20,714 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f64cb2042c701322ac00e312f65f16, ASSIGN 2024-11-25T03:41:20,714 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b224c7c31e6ff664ce67ac884f573d7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:20,714 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b224c7c31e6ff664ce67ac884f573d7, ASSIGN 2024-11-25T03:41:20,714 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b224c7c31e6ff664ce67ac884f573d7, server=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:20,714 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:20,714 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8b5ae4b02f22b24f40505bb660957004, server=d73269e94d72,38397,1732506072528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:20,714 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure c769ca025a015554efcc98619008be96, server=d73269e94d72,38397,1732506072528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:20,714 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5f64cb2042c701322ac00e312f65f16, server=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:20,714 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 922ebf14d68aba40da3a2e99777c777b, server=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:20,714 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=81, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4705fa4ae38907dd8b9783be2dff29da, server=d73269e94d72,38397,1732506072528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:20,714 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=82, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 857a6b6999e909a9b73e698c69291205, server=d73269e94d72,38397,1732506072528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:20,715 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 15 msec 2024-11-25T03:41:20,715 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40325 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T03:41:20,716 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40325 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T03:41:20,717 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=65 2024-11-25T03:41:20,717 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=d73269e94d72,33145,1732506072477, table=hbase:meta, region=1588230740 2024-11-25T03:41:20,718 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d73269e94d72,33145,1732506072477, state=OPEN 2024-11-25T03:41:20,776 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38397-0x1017014aa590003, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:20,776 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40325-0x1017014aa590000, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:20,776 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33145-0x1017014aa590002, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:20,777 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:20,777 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:20,777 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:20,778 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN to state=OFFLINE, location=null, table=test, region=857a6b6999e909a9b73e698c69291205 to restore RIT 2024-11-25T03:41:20,778 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN to state=OFFLINE, location=null, table=test, region=3b442507eaac3b5222a3c4f97ff80208 to restore RIT 2024-11-25T03:41:20,779 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN to state=OFFLINE, location=null, table=test, region=922ebf14d68aba40da3a2e99777c777b to restore RIT 2024-11-25T03:41:20,779 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4705fa4ae38907dd8b9783be2dff29da, ASSIGN to state=OFFLINE, location=null, table=test, region=4705fa4ae38907dd8b9783be2dff29da to restore RIT 2024-11-25T03:41:20,779 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c769ca025a015554efcc98619008be96, ASSIGN to state=OFFLINE, location=null, table=test, region=c769ca025a015554efcc98619008be96 to restore RIT 2024-11-25T03:41:20,779 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b5ae4b02f22b24f40505bb660957004, ASSIGN to state=OFFLINE, location=null, table=test, region=8b5ae4b02f22b24f40505bb660957004 to restore RIT 2024-11-25T03:41:20,779 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f64cb2042c701322ac00e312f65f16, ASSIGN to state=OFFLINE, location=null, table=test, region=f5f64cb2042c701322ac00e312f65f16 to restore RIT 2024-11-25T03:41:20,779 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b224c7c31e6ff664ce67ac884f573d7, ASSIGN to state=OFFLINE, location=null, table=test, region=2b224c7c31e6ff664ce67ac884f573d7 to restore RIT 2024-11-25T03:41:20,780 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732506110780 2024-11-25T03:41:20,780 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-25T03:41:20,780 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T03:41:20,783 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-25T03:41:20,785 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-25T03:41:20,785 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=81, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4705fa4ae38907dd8b9783be2dff29da, server=d73269e94d72,38397,1732506072528 2024-11-25T03:41:20,785 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T03:41:20,785 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=81, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4705fa4ae38907dd8b9783be2dff29da, server=d73269e94d72,38397,1732506072528 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T03:41:20,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3b442507eaac3b5222a3c4f97ff80208, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:20,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c73ee0e3b504288792edda3ec9a26049, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:20,809 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e3a490d70b339ac73875ea7638f934f, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:20,809 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85a5fab26752f98e6966cd407e31db69, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:20,809 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a7d4164efdaed87f07b6f76a94819b3, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:20,809 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bba1924bf10ad03aeb55628aeffc2363, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:20,809 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=857a6b6999e909a9b73e698c69291205, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:20,810 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e791add66a4fee50be34aa83a6d502a, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:20,810 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c0b379dd00b53bde73bca0e12f321ce, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:20,810 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5f64cb2042c701322ac00e312f65f16, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:20,810 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4b8682bf62ac734aa4174eba4ee1ddc, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:20,810 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb9ff3be6d639a626d88387efa9120fd, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:20,810 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0c260efb5b2a8a6137010d439efc8d47, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:20,811 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4705fa4ae38907dd8b9783be2dff29da, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:20,811 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5da71c9c32aa33cf6f067ea6d82ae84f, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:20,811 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fc93e882c2b7c5acdb53e9cf1071dc8, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:20,811 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=922ebf14d68aba40da3a2e99777c777b, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:20,811 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=745356a682ec14d9f6e2066d456cab06, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:20,812 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0698446036b7ae35179ebc9b07274db5, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:20,812 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b5ae4b02f22b24f40505bb660957004, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:20,812 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6aed1980a8b63b44c776d0990f923aa7, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:20,812 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=706facb174cc9547b71f08aa672ddcee, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:20,812 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b224c7c31e6ff664ce67ac884f573d7, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:20,812 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f812dacea03120e7730979dc37e38c6, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:20,813 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df71ad9490ccea77dfed73e590672db8, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:20,813 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c769ca025a015554efcc98619008be96, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:20,813 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-25T03:41:20,813 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732506140813 2024-11-25T03:41:20,813 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732506200813 2024-11-25T03:41:20,813 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 33 msec 2024-11-25T03:41:20,813 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-25T03:41:20,813 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T03:41:20,813 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-25T03:41:20,813 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-25T03:41:20,814 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-25T03:41:20,814 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-25T03:41:20,814 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-25T03:41:20,814 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-25T03:41:20,815 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-25T03:41:20,819 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-25T03:41:20,819 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-25T03:41:20,819 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-25T03:41:20,819 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:20,819 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:20,820 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-25T03:41:20,820 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:20,820 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-25T03:41:20,820 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:20,820 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=81, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-25T03:41:20,820 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:20,820 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-25T03:41:20,820 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:20,820 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-25T03:41:20,820 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:20,820 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-25T03:41:20,820 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:20,820 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-25T03:41:20,820 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-25T03:41:20,820 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-25T03:41:20,820 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40325 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T03:41:20,820 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:20,821 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-25T03:41:20,821 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-25T03:41:20,821 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40325 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T03:41:20,822 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-25T03:41:20,823 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, REOPEN/MOVE 2024-11-25T03:41:20,825 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true 2024-11-25T03:41:20,825 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 2024-11-25T03:41:20,826 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 2024-11-25T03:41:20,826 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T03:41:20,826 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d73269e94d72,33145,1732506072477 2024-11-25T03:41:20,826 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 2024-11-25T03:41:20,826 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 2024-11-25T03:41:20,826 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN 2024-11-25T03:41:20,826 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN 2024-11-25T03:41:20,826 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN 2024-11-25T03:41:20,826 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4705fa4ae38907dd8b9783be2dff29da, ASSIGN 2024-11-25T03:41:20,826 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c769ca025a015554efcc98619008be96, ASSIGN 2024-11-25T03:41:20,826 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b5ae4b02f22b24f40505bb660957004, ASSIGN 2024-11-25T03:41:20,826 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f64cb2042c701322ac00e312f65f16, ASSIGN 2024-11-25T03:41:20,826 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b224c7c31e6ff664ce67ac884f573d7, ASSIGN 2024-11-25T03:41:20,826 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b224c7c31e6ff664ce67ac884f573d7, server=d73269e94d72,33145,1732506072477 2024-11-25T03:41:20,826 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477 2024-11-25T03:41:20,826 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8b5ae4b02f22b24f40505bb660957004, server=d73269e94d72,38397,1732506072528 2024-11-25T03:41:20,826 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure c769ca025a015554efcc98619008be96, server=d73269e94d72,38397,1732506072528 2024-11-25T03:41:20,826 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5f64cb2042c701322ac00e312f65f16, server=d73269e94d72,33145,1732506072477 2024-11-25T03:41:20,826 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 922ebf14d68aba40da3a2e99777c777b, server=d73269e94d72,33145,1732506072477 2024-11-25T03:41:20,826 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=81, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4705fa4ae38907dd8b9783be2dff29da, server=d73269e94d72,38397,1732506072528 2024-11-25T03:41:20,826 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:20,826 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:20,826 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:20,826 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:20,826 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:20,826 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:20,826 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:20,826 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:20,826 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN 2024-11-25T03:41:20,826 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:20,826 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN 2024-11-25T03:41:20,826 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:20,827 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN 2024-11-25T03:41:20,827 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4705fa4ae38907dd8b9783be2dff29da, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:20,827 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4705fa4ae38907dd8b9783be2dff29da, ASSIGN 2024-11-25T03:41:20,827 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c769ca025a015554efcc98619008be96, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:20,827 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c769ca025a015554efcc98619008be96, ASSIGN 2024-11-25T03:41:20,827 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b5ae4b02f22b24f40505bb660957004, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:20,827 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b5ae4b02f22b24f40505bb660957004, ASSIGN 2024-11-25T03:41:20,827 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f64cb2042c701322ac00e312f65f16, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:20,827 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f64cb2042c701322ac00e312f65f16, ASSIGN 2024-11-25T03:41:20,827 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b224c7c31e6ff664ce67ac884f573d7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:20,827 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b224c7c31e6ff664ce67ac884f573d7, ASSIGN 2024-11-25T03:41:20,827 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b224c7c31e6ff664ce67ac884f573d7, server=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:20,827 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:20,827 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8b5ae4b02f22b24f40505bb660957004, server=d73269e94d72,38397,1732506072528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:20,827 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure c769ca025a015554efcc98619008be96, server=d73269e94d72,38397,1732506072528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:20,827 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5f64cb2042c701322ac00e312f65f16, server=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:20,827 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 922ebf14d68aba40da3a2e99777c777b, server=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:20,827 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=81, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4705fa4ae38907dd8b9783be2dff29da, server=d73269e94d72,38397,1732506072528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:20,827 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 12 msec 2024-11-25T03:41:20,828 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=65 2024-11-25T03:41:20,828 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=d73269e94d72,33145,1732506072477, table=hbase:meta, region=1588230740 2024-11-25T03:41:20,828 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d73269e94d72,33145,1732506072477, state=OPEN 2024-11-25T03:41:20,840 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38397-0x1017014aa590003, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:20,840 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33145-0x1017014aa590002, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:20,840 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40325-0x1017014aa590000, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:20,840 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:20,840 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:20,840 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:20,840 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN to state=OFFLINE, location=null, table=test, region=857a6b6999e909a9b73e698c69291205 to restore RIT 2024-11-25T03:41:20,840 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN to state=OFFLINE, location=null, table=test, region=3b442507eaac3b5222a3c4f97ff80208 to restore RIT 2024-11-25T03:41:20,841 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN to state=OFFLINE, location=null, table=test, region=922ebf14d68aba40da3a2e99777c777b to restore RIT 2024-11-25T03:41:20,841 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4705fa4ae38907dd8b9783be2dff29da, ASSIGN to state=OFFLINE, location=null, table=test, region=4705fa4ae38907dd8b9783be2dff29da to restore RIT 2024-11-25T03:41:20,841 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c769ca025a015554efcc98619008be96, ASSIGN to state=OFFLINE, location=null, table=test, region=c769ca025a015554efcc98619008be96 to restore RIT 2024-11-25T03:41:20,841 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b5ae4b02f22b24f40505bb660957004, ASSIGN to state=OFFLINE, location=null, table=test, region=8b5ae4b02f22b24f40505bb660957004 to restore RIT 2024-11-25T03:41:20,841 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f64cb2042c701322ac00e312f65f16, ASSIGN to state=OFFLINE, location=null, table=test, region=f5f64cb2042c701322ac00e312f65f16 to restore RIT 2024-11-25T03:41:20,841 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b224c7c31e6ff664ce67ac884f573d7, ASSIGN to state=OFFLINE, location=null, table=test, region=2b224c7c31e6ff664ce67ac884f573d7 to restore RIT 2024-11-25T03:41:20,841 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732506110841 2024-11-25T03:41:20,841 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-25T03:41:20,841 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T03:41:20,844 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-25T03:41:20,845 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-25T03:41:20,845 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=80, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 922ebf14d68aba40da3a2e99777c777b, server=d73269e94d72,33145,1732506072477 2024-11-25T03:41:20,845 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T03:41:20,845 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=80, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 922ebf14d68aba40da3a2e99777c777b, server=d73269e94d72,33145,1732506072477 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T03:41:20,852 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3b442507eaac3b5222a3c4f97ff80208, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:20,852 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c73ee0e3b504288792edda3ec9a26049, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:20,853 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e3a490d70b339ac73875ea7638f934f, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:20,853 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85a5fab26752f98e6966cd407e31db69, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:20,853 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a7d4164efdaed87f07b6f76a94819b3, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:20,853 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bba1924bf10ad03aeb55628aeffc2363, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:20,853 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=857a6b6999e909a9b73e698c69291205, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:20,854 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e791add66a4fee50be34aa83a6d502a, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:20,854 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c0b379dd00b53bde73bca0e12f321ce, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:20,854 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5f64cb2042c701322ac00e312f65f16, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:20,854 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4b8682bf62ac734aa4174eba4ee1ddc, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:20,854 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb9ff3be6d639a626d88387efa9120fd, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:20,854 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0c260efb5b2a8a6137010d439efc8d47, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:20,854 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4705fa4ae38907dd8b9783be2dff29da, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:20,855 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5da71c9c32aa33cf6f067ea6d82ae84f, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:20,855 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fc93e882c2b7c5acdb53e9cf1071dc8, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:20,855 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=922ebf14d68aba40da3a2e99777c777b, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:20,855 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=745356a682ec14d9f6e2066d456cab06, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:20,855 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0698446036b7ae35179ebc9b07274db5, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:20,855 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b5ae4b02f22b24f40505bb660957004, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:20,855 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6aed1980a8b63b44c776d0990f923aa7, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:20,856 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=706facb174cc9547b71f08aa672ddcee, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:20,856 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b224c7c31e6ff664ce67ac884f573d7, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:20,856 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f812dacea03120e7730979dc37e38c6, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:20,856 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df71ad9490ccea77dfed73e590672db8, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:20,856 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c769ca025a015554efcc98619008be96, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:20,856 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-25T03:41:20,856 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732506140856 2024-11-25T03:41:20,856 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732506200856 2024-11-25T03:41:20,856 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 15 msec 2024-11-25T03:41:20,856 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-25T03:41:20,856 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T03:41:20,857 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-25T03:41:20,857 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-25T03:41:20,857 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-25T03:41:20,857 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-25T03:41:20,857 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-25T03:41:20,857 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-25T03:41:20,858 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-25T03:41:20,862 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-25T03:41:20,862 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-25T03:41:20,862 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-25T03:41:20,862 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:20,862 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:20,862 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-25T03:41:20,862 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:20,862 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-25T03:41:20,862 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:20,862 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:20,862 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-25T03:41:20,862 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:20,862 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-25T03:41:20,862 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:20,862 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-25T03:41:20,862 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:20,862 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-25T03:41:20,862 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-25T03:41:20,862 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-25T03:41:20,862 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:20,862 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-25T03:41:20,863 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-25T03:41:20,864 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-25T03:41:20,864 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, REOPEN/MOVE 2024-11-25T03:41:20,866 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true 2024-11-25T03:41:20,866 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 2024-11-25T03:41:20,866 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 2024-11-25T03:41:20,866 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T03:41:20,866 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d73269e94d72,33145,1732506072477 2024-11-25T03:41:20,866 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 2024-11-25T03:41:20,866 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 2024-11-25T03:41:20,866 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN 2024-11-25T03:41:20,866 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN 2024-11-25T03:41:20,866 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN 2024-11-25T03:41:20,866 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4705fa4ae38907dd8b9783be2dff29da, ASSIGN 2024-11-25T03:41:20,866 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c769ca025a015554efcc98619008be96, ASSIGN 2024-11-25T03:41:20,866 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b5ae4b02f22b24f40505bb660957004, ASSIGN 2024-11-25T03:41:20,866 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f64cb2042c701322ac00e312f65f16, ASSIGN 2024-11-25T03:41:20,866 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b224c7c31e6ff664ce67ac884f573d7, ASSIGN 2024-11-25T03:41:20,866 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b224c7c31e6ff664ce67ac884f573d7, server=d73269e94d72,33145,1732506072477 2024-11-25T03:41:20,866 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477 2024-11-25T03:41:20,866 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8b5ae4b02f22b24f40505bb660957004, server=d73269e94d72,38397,1732506072528 2024-11-25T03:41:20,866 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure c769ca025a015554efcc98619008be96, server=d73269e94d72,38397,1732506072528 2024-11-25T03:41:20,866 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5f64cb2042c701322ac00e312f65f16, server=d73269e94d72,33145,1732506072477 2024-11-25T03:41:20,866 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 922ebf14d68aba40da3a2e99777c777b, server=d73269e94d72,33145,1732506072477 2024-11-25T03:41:20,866 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:20,867 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:20,867 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:20,867 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:20,867 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:20,867 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:20,867 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:20,867 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:20,867 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN 2024-11-25T03:41:20,867 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:20,867 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN 2024-11-25T03:41:20,867 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:20,867 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN 2024-11-25T03:41:20,867 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4705fa4ae38907dd8b9783be2dff29da, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:20,867 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4705fa4ae38907dd8b9783be2dff29da, ASSIGN 2024-11-25T03:41:20,867 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c769ca025a015554efcc98619008be96, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:20,867 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c769ca025a015554efcc98619008be96, ASSIGN 2024-11-25T03:41:20,867 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b5ae4b02f22b24f40505bb660957004, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:20,867 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b5ae4b02f22b24f40505bb660957004, ASSIGN 2024-11-25T03:41:20,867 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f64cb2042c701322ac00e312f65f16, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:20,867 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f64cb2042c701322ac00e312f65f16, ASSIGN 2024-11-25T03:41:20,867 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b224c7c31e6ff664ce67ac884f573d7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:20,867 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b224c7c31e6ff664ce67ac884f573d7, ASSIGN 2024-11-25T03:41:20,867 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b224c7c31e6ff664ce67ac884f573d7, server=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:20,867 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:20,867 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8b5ae4b02f22b24f40505bb660957004, server=d73269e94d72,38397,1732506072528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:20,867 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure c769ca025a015554efcc98619008be96, server=d73269e94d72,38397,1732506072528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:20,867 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5f64cb2042c701322ac00e312f65f16, server=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:20,868 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 922ebf14d68aba40da3a2e99777c777b, server=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:20,868 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-25T03:41:20,869 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=65 2024-11-25T03:41:20,869 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=d73269e94d72,33145,1732506072477, table=hbase:meta, region=1588230740 2024-11-25T03:41:20,869 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d73269e94d72,33145,1732506072477, state=OPEN 2024-11-25T03:41:20,924 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40325-0x1017014aa590000, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:20,924 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33145-0x1017014aa590002, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:20,924 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:20,924 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:20,924 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38397-0x1017014aa590003, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:20,924 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40325 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T03:41:20,924 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:20,925 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN to state=OFFLINE, location=null, table=test, region=857a6b6999e909a9b73e698c69291205 to restore RIT 2024-11-25T03:41:20,925 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40325 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T03:41:20,925 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN to state=OFFLINE, location=null, table=test, region=3b442507eaac3b5222a3c4f97ff80208 to restore RIT 2024-11-25T03:41:20,925 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN to state=OFFLINE, location=null, table=test, region=922ebf14d68aba40da3a2e99777c777b to restore RIT 2024-11-25T03:41:20,925 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4705fa4ae38907dd8b9783be2dff29da, ASSIGN to state=OFFLINE, location=null, table=test, region=4705fa4ae38907dd8b9783be2dff29da to restore RIT 2024-11-25T03:41:20,925 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c769ca025a015554efcc98619008be96, ASSIGN to state=OFFLINE, location=null, table=test, region=c769ca025a015554efcc98619008be96 to restore RIT 2024-11-25T03:41:20,925 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b5ae4b02f22b24f40505bb660957004, ASSIGN to state=OFFLINE, location=null, table=test, region=8b5ae4b02f22b24f40505bb660957004 to restore RIT 2024-11-25T03:41:20,925 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f64cb2042c701322ac00e312f65f16, ASSIGN to state=OFFLINE, location=null, table=test, region=f5f64cb2042c701322ac00e312f65f16 to restore RIT 2024-11-25T03:41:20,925 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b224c7c31e6ff664ce67ac884f573d7, ASSIGN to state=OFFLINE, location=null, table=test, region=2b224c7c31e6ff664ce67ac884f573d7 to restore RIT 2024-11-25T03:41:20,926 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732506110926 2024-11-25T03:41:20,926 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-25T03:41:20,926 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T03:41:20,927 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-25T03:41:20,928 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-25T03:41:20,928 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5f64cb2042c701322ac00e312f65f16, server=d73269e94d72,33145,1732506072477 2024-11-25T03:41:20,928 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T03:41:20,928 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5f64cb2042c701322ac00e312f65f16, server=d73269e94d72,33145,1732506072477 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T03:41:20,936 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3b442507eaac3b5222a3c4f97ff80208, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:20,936 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c73ee0e3b504288792edda3ec9a26049, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:20,936 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e3a490d70b339ac73875ea7638f934f, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:20,936 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85a5fab26752f98e6966cd407e31db69, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:20,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a7d4164efdaed87f07b6f76a94819b3, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:20,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bba1924bf10ad03aeb55628aeffc2363, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:20,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=857a6b6999e909a9b73e698c69291205, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:20,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e791add66a4fee50be34aa83a6d502a, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:20,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c0b379dd00b53bde73bca0e12f321ce, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:20,938 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5f64cb2042c701322ac00e312f65f16, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:20,938 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4b8682bf62ac734aa4174eba4ee1ddc, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:20,938 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb9ff3be6d639a626d88387efa9120fd, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:20,938 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0c260efb5b2a8a6137010d439efc8d47, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:20,938 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4705fa4ae38907dd8b9783be2dff29da, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:20,939 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5da71c9c32aa33cf6f067ea6d82ae84f, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:20,939 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fc93e882c2b7c5acdb53e9cf1071dc8, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:20,939 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=922ebf14d68aba40da3a2e99777c777b, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:20,939 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=745356a682ec14d9f6e2066d456cab06, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:20,939 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0698446036b7ae35179ebc9b07274db5, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:20,940 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b5ae4b02f22b24f40505bb660957004, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:20,940 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6aed1980a8b63b44c776d0990f923aa7, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:20,940 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=706facb174cc9547b71f08aa672ddcee, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:20,940 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b224c7c31e6ff664ce67ac884f573d7, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:20,940 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f812dacea03120e7730979dc37e38c6, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:20,941 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df71ad9490ccea77dfed73e590672db8, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:20,941 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c769ca025a015554efcc98619008be96, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:20,941 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-25T03:41:20,941 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732506140941 2024-11-25T03:41:20,941 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732506200941 2024-11-25T03:41:20,941 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 15 msec 2024-11-25T03:41:20,941 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-25T03:41:20,941 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T03:41:20,941 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-25T03:41:20,941 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-25T03:41:20,942 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-25T03:41:20,942 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-25T03:41:20,942 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-25T03:41:20,942 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-25T03:41:20,943 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-25T03:41:20,948 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-25T03:41:20,948 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-25T03:41:20,948 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-25T03:41:20,948 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:20,948 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:20,948 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-25T03:41:20,948 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:20,949 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:20,949 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:20,949 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-25T03:41:20,949 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:20,949 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-25T03:41:20,949 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:20,949 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-25T03:41:20,949 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:20,949 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-25T03:41:20,949 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-25T03:41:20,949 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-25T03:41:20,949 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:20,949 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-25T03:41:20,949 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-25T03:41:20,951 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-25T03:41:20,951 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, REOPEN/MOVE 2024-11-25T03:41:20,954 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true 2024-11-25T03:41:20,954 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 2024-11-25T03:41:20,954 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 2024-11-25T03:41:20,954 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T03:41:20,954 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d73269e94d72,33145,1732506072477 2024-11-25T03:41:20,954 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 2024-11-25T03:41:20,954 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 2024-11-25T03:41:20,954 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN 2024-11-25T03:41:20,954 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN 2024-11-25T03:41:20,954 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN 2024-11-25T03:41:20,954 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4705fa4ae38907dd8b9783be2dff29da, ASSIGN 2024-11-25T03:41:20,954 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c769ca025a015554efcc98619008be96, ASSIGN 2024-11-25T03:41:20,954 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b5ae4b02f22b24f40505bb660957004, ASSIGN 2024-11-25T03:41:20,954 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f64cb2042c701322ac00e312f65f16, ASSIGN 2024-11-25T03:41:20,954 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b224c7c31e6ff664ce67ac884f573d7, ASSIGN 2024-11-25T03:41:20,954 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b224c7c31e6ff664ce67ac884f573d7, server=d73269e94d72,33145,1732506072477 2024-11-25T03:41:20,954 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477 2024-11-25T03:41:20,954 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8b5ae4b02f22b24f40505bb660957004, server=d73269e94d72,38397,1732506072528 2024-11-25T03:41:20,954 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure c769ca025a015554efcc98619008be96, server=d73269e94d72,38397,1732506072528 2024-11-25T03:41:20,955 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5f64cb2042c701322ac00e312f65f16, server=d73269e94d72,33145,1732506072477 2024-11-25T03:41:20,955 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:20,955 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:20,955 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:20,955 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:20,955 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:20,955 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:20,955 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:20,955 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:20,955 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN 2024-11-25T03:41:20,955 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:20,955 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN 2024-11-25T03:41:20,955 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:20,955 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN 2024-11-25T03:41:20,955 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4705fa4ae38907dd8b9783be2dff29da, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:20,955 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4705fa4ae38907dd8b9783be2dff29da, ASSIGN 2024-11-25T03:41:20,955 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c769ca025a015554efcc98619008be96, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:20,955 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c769ca025a015554efcc98619008be96, ASSIGN 2024-11-25T03:41:20,955 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b5ae4b02f22b24f40505bb660957004, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:20,955 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b5ae4b02f22b24f40505bb660957004, ASSIGN 2024-11-25T03:41:20,955 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f64cb2042c701322ac00e312f65f16, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:20,955 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f64cb2042c701322ac00e312f65f16, ASSIGN 2024-11-25T03:41:20,956 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b224c7c31e6ff664ce67ac884f573d7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:20,956 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b224c7c31e6ff664ce67ac884f573d7, ASSIGN 2024-11-25T03:41:20,956 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b224c7c31e6ff664ce67ac884f573d7, server=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:20,956 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:20,956 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8b5ae4b02f22b24f40505bb660957004, server=d73269e94d72,38397,1732506072528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:20,956 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure c769ca025a015554efcc98619008be96, server=d73269e94d72,38397,1732506072528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:20,956 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5f64cb2042c701322ac00e312f65f16, server=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:20,956 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 13 msec 2024-11-25T03:41:20,957 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=65 2024-11-25T03:41:20,957 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=d73269e94d72,33145,1732506072477, table=hbase:meta, region=1588230740 2024-11-25T03:41:20,958 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d73269e94d72,33145,1732506072477, state=OPEN 2024-11-25T03:41:20,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38397-0x1017014aa590003, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:20,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40325-0x1017014aa590000, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:20,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33145-0x1017014aa590002, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:20,998 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:20,998 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:20,998 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:20,999 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN to state=OFFLINE, location=null, table=test, region=857a6b6999e909a9b73e698c69291205 to restore RIT 2024-11-25T03:41:20,999 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN to state=OFFLINE, location=null, table=test, region=3b442507eaac3b5222a3c4f97ff80208 to restore RIT 2024-11-25T03:41:20,999 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN to state=OFFLINE, location=null, table=test, region=922ebf14d68aba40da3a2e99777c777b to restore RIT 2024-11-25T03:41:20,999 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4705fa4ae38907dd8b9783be2dff29da, ASSIGN to state=OFFLINE, location=null, table=test, region=4705fa4ae38907dd8b9783be2dff29da to restore RIT 2024-11-25T03:41:20,999 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c769ca025a015554efcc98619008be96, ASSIGN to state=OFFLINE, location=null, table=test, region=c769ca025a015554efcc98619008be96 to restore RIT 2024-11-25T03:41:20,999 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b5ae4b02f22b24f40505bb660957004, ASSIGN to state=OFFLINE, location=null, table=test, region=8b5ae4b02f22b24f40505bb660957004 to restore RIT 2024-11-25T03:41:20,999 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f64cb2042c701322ac00e312f65f16, ASSIGN to state=OFFLINE, location=null, table=test, region=f5f64cb2042c701322ac00e312f65f16 to restore RIT 2024-11-25T03:41:20,999 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b224c7c31e6ff664ce67ac884f573d7, ASSIGN to state=OFFLINE, location=null, table=test, region=2b224c7c31e6ff664ce67ac884f573d7 to restore RIT 2024-11-25T03:41:20,999 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732506110999 2024-11-25T03:41:20,999 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-25T03:41:20,999 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T03:41:21,000 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-25T03:41:21,002 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-25T03:41:21,002 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure c769ca025a015554efcc98619008be96, server=d73269e94d72,38397,1732506072528 2024-11-25T03:41:21,002 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T03:41:21,002 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure c769ca025a015554efcc98619008be96, server=d73269e94d72,38397,1732506072528 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T03:41:21,006 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3b442507eaac3b5222a3c4f97ff80208, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,006 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c73ee0e3b504288792edda3ec9a26049, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,006 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e3a490d70b339ac73875ea7638f934f, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,006 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85a5fab26752f98e6966cd407e31db69, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,006 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a7d4164efdaed87f07b6f76a94819b3, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,006 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bba1924bf10ad03aeb55628aeffc2363, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,007 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=857a6b6999e909a9b73e698c69291205, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,007 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e791add66a4fee50be34aa83a6d502a, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,007 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c0b379dd00b53bde73bca0e12f321ce, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,007 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5f64cb2042c701322ac00e312f65f16, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,007 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4b8682bf62ac734aa4174eba4ee1ddc, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,007 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb9ff3be6d639a626d88387efa9120fd, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,007 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0c260efb5b2a8a6137010d439efc8d47, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,007 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4705fa4ae38907dd8b9783be2dff29da, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,008 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5da71c9c32aa33cf6f067ea6d82ae84f, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,008 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fc93e882c2b7c5acdb53e9cf1071dc8, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,008 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=922ebf14d68aba40da3a2e99777c777b, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,008 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=745356a682ec14d9f6e2066d456cab06, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,008 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0698446036b7ae35179ebc9b07274db5, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,008 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b5ae4b02f22b24f40505bb660957004, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,008 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6aed1980a8b63b44c776d0990f923aa7, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,009 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=706facb174cc9547b71f08aa672ddcee, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,009 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b224c7c31e6ff664ce67ac884f573d7, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,009 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f812dacea03120e7730979dc37e38c6, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,009 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df71ad9490ccea77dfed73e590672db8, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,009 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c769ca025a015554efcc98619008be96, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,009 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-25T03:41:21,009 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732506141009 2024-11-25T03:41:21,009 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732506201009 2024-11-25T03:41:21,009 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-25T03:41:21,009 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-25T03:41:21,009 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T03:41:21,009 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-25T03:41:21,010 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-25T03:41:21,010 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-25T03:41:21,010 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-25T03:41:21,010 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-25T03:41:21,010 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-25T03:41:21,011 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-25T03:41:21,014 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-25T03:41:21,014 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-25T03:41:21,014 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-25T03:41:21,014 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,014 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,014 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-25T03:41:21,014 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,014 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,014 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,014 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-25T03:41:21,014 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,015 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-25T03:41:21,015 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,015 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,015 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-25T03:41:21,015 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-25T03:41:21,015 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-25T03:41:21,015 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,015 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-25T03:41:21,015 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-25T03:41:21,016 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-25T03:41:21,016 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, REOPEN/MOVE 2024-11-25T03:41:21,018 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true 2024-11-25T03:41:21,018 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 2024-11-25T03:41:21,018 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 2024-11-25T03:41:21,018 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T03:41:21,018 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d73269e94d72,33145,1732506072477 2024-11-25T03:41:21,018 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 2024-11-25T03:41:21,018 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 2024-11-25T03:41:21,018 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN 2024-11-25T03:41:21,018 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN 2024-11-25T03:41:21,018 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN 2024-11-25T03:41:21,018 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4705fa4ae38907dd8b9783be2dff29da, ASSIGN 2024-11-25T03:41:21,018 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c769ca025a015554efcc98619008be96, ASSIGN 2024-11-25T03:41:21,018 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b5ae4b02f22b24f40505bb660957004, ASSIGN 2024-11-25T03:41:21,018 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f64cb2042c701322ac00e312f65f16, ASSIGN 2024-11-25T03:41:21,018 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b224c7c31e6ff664ce67ac884f573d7, ASSIGN 2024-11-25T03:41:21,018 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b224c7c31e6ff664ce67ac884f573d7, server=d73269e94d72,33145,1732506072477 2024-11-25T03:41:21,018 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477 2024-11-25T03:41:21,018 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8b5ae4b02f22b24f40505bb660957004, server=d73269e94d72,38397,1732506072528 2024-11-25T03:41:21,018 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure c769ca025a015554efcc98619008be96, server=d73269e94d72,38397,1732506072528 2024-11-25T03:41:21,018 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,018 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,018 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,018 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,018 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,018 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,018 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,018 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,018 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN 2024-11-25T03:41:21,019 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,019 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN 2024-11-25T03:41:21,019 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,019 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN 2024-11-25T03:41:21,019 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4705fa4ae38907dd8b9783be2dff29da, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,019 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4705fa4ae38907dd8b9783be2dff29da, ASSIGN 2024-11-25T03:41:21,019 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c769ca025a015554efcc98619008be96, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,019 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c769ca025a015554efcc98619008be96, ASSIGN 2024-11-25T03:41:21,019 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b5ae4b02f22b24f40505bb660957004, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,019 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b5ae4b02f22b24f40505bb660957004, ASSIGN 2024-11-25T03:41:21,019 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f64cb2042c701322ac00e312f65f16, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,019 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f64cb2042c701322ac00e312f65f16, ASSIGN 2024-11-25T03:41:21,019 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b224c7c31e6ff664ce67ac884f573d7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,019 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b224c7c31e6ff664ce67ac884f573d7, ASSIGN 2024-11-25T03:41:21,019 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b224c7c31e6ff664ce67ac884f573d7, server=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,019 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,019 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8b5ae4b02f22b24f40505bb660957004, server=d73269e94d72,38397,1732506072528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,019 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure c769ca025a015554efcc98619008be96, server=d73269e94d72,38397,1732506072528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,019 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-11-25T03:41:21,020 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=65 2024-11-25T03:41:21,020 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=d73269e94d72,33145,1732506072477, table=hbase:meta, region=1588230740 2024-11-25T03:41:21,020 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d73269e94d72,33145,1732506072477, state=OPEN 2024-11-25T03:41:21,027 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40325 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T03:41:21,028 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40325 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T03:41:21,029 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40325-0x1017014aa590000, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:21,029 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33145-0x1017014aa590002, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:21,029 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38397-0x1017014aa590003, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:21,029 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:21,029 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:21,029 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:21,030 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN to state=OFFLINE, location=null, table=test, region=857a6b6999e909a9b73e698c69291205 to restore RIT 2024-11-25T03:41:21,030 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN to state=OFFLINE, location=null, table=test, region=3b442507eaac3b5222a3c4f97ff80208 to restore RIT 2024-11-25T03:41:21,030 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN to state=OFFLINE, location=null, table=test, region=922ebf14d68aba40da3a2e99777c777b to restore RIT 2024-11-25T03:41:21,030 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4705fa4ae38907dd8b9783be2dff29da, ASSIGN to state=OFFLINE, location=null, table=test, region=4705fa4ae38907dd8b9783be2dff29da to restore RIT 2024-11-25T03:41:21,030 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c769ca025a015554efcc98619008be96, ASSIGN to state=OFFLINE, location=null, table=test, region=c769ca025a015554efcc98619008be96 to restore RIT 2024-11-25T03:41:21,030 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b5ae4b02f22b24f40505bb660957004, ASSIGN to state=OFFLINE, location=null, table=test, region=8b5ae4b02f22b24f40505bb660957004 to restore RIT 2024-11-25T03:41:21,031 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f64cb2042c701322ac00e312f65f16, ASSIGN to state=OFFLINE, location=null, table=test, region=f5f64cb2042c701322ac00e312f65f16 to restore RIT 2024-11-25T03:41:21,031 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b224c7c31e6ff664ce67ac884f573d7, ASSIGN to state=OFFLINE, location=null, table=test, region=2b224c7c31e6ff664ce67ac884f573d7 to restore RIT 2024-11-25T03:41:21,032 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732506111032 2024-11-25T03:41:21,032 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-25T03:41:21,032 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T03:41:21,034 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-25T03:41:21,035 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-25T03:41:21,035 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=77, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8b5ae4b02f22b24f40505bb660957004, server=d73269e94d72,38397,1732506072528 2024-11-25T03:41:21,035 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T03:41:21,035 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=77, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8b5ae4b02f22b24f40505bb660957004, server=d73269e94d72,38397,1732506072528 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T03:41:21,040 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3b442507eaac3b5222a3c4f97ff80208, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,041 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c73ee0e3b504288792edda3ec9a26049, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,041 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e3a490d70b339ac73875ea7638f934f, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,041 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85a5fab26752f98e6966cd407e31db69, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,042 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a7d4164efdaed87f07b6f76a94819b3, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,042 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bba1924bf10ad03aeb55628aeffc2363, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,042 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=857a6b6999e909a9b73e698c69291205, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,042 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e791add66a4fee50be34aa83a6d502a, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,042 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c0b379dd00b53bde73bca0e12f321ce, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,043 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5f64cb2042c701322ac00e312f65f16, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,043 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4b8682bf62ac734aa4174eba4ee1ddc, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,043 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb9ff3be6d639a626d88387efa9120fd, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,043 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0c260efb5b2a8a6137010d439efc8d47, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,043 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4705fa4ae38907dd8b9783be2dff29da, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,044 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5da71c9c32aa33cf6f067ea6d82ae84f, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,044 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fc93e882c2b7c5acdb53e9cf1071dc8, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,044 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=922ebf14d68aba40da3a2e99777c777b, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,044 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=745356a682ec14d9f6e2066d456cab06, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,044 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0698446036b7ae35179ebc9b07274db5, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,045 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b5ae4b02f22b24f40505bb660957004, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,045 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6aed1980a8b63b44c776d0990f923aa7, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,045 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=706facb174cc9547b71f08aa672ddcee, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,045 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b224c7c31e6ff664ce67ac884f573d7, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,045 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f812dacea03120e7730979dc37e38c6, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,045 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df71ad9490ccea77dfed73e590672db8, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,046 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c769ca025a015554efcc98619008be96, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,046 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-25T03:41:21,046 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732506141046 2024-11-25T03:41:21,046 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732506201046 2024-11-25T03:41:21,046 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 13 msec 2024-11-25T03:41:21,046 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-25T03:41:21,046 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T03:41:21,046 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-25T03:41:21,046 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-25T03:41:21,047 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-25T03:41:21,047 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-25T03:41:21,047 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-25T03:41:21,047 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-25T03:41:21,048 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-25T03:41:21,052 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-25T03:41:21,052 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-25T03:41:21,052 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-25T03:41:21,052 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,052 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,052 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-25T03:41:21,052 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,052 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,052 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,052 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,053 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-25T03:41:21,053 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,053 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,053 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-25T03:41:21,053 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-25T03:41:21,053 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-25T03:41:21,053 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,053 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-25T03:41:21,053 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-25T03:41:21,054 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-25T03:41:21,055 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, REOPEN/MOVE 2024-11-25T03:41:21,057 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true 2024-11-25T03:41:21,057 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 2024-11-25T03:41:21,057 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 2024-11-25T03:41:21,057 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T03:41:21,057 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d73269e94d72,33145,1732506072477 2024-11-25T03:41:21,057 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 2024-11-25T03:41:21,058 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 2024-11-25T03:41:21,058 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN 2024-11-25T03:41:21,058 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN 2024-11-25T03:41:21,058 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN 2024-11-25T03:41:21,058 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4705fa4ae38907dd8b9783be2dff29da, ASSIGN 2024-11-25T03:41:21,058 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c769ca025a015554efcc98619008be96, ASSIGN 2024-11-25T03:41:21,058 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b5ae4b02f22b24f40505bb660957004, ASSIGN 2024-11-25T03:41:21,058 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f64cb2042c701322ac00e312f65f16, ASSIGN 2024-11-25T03:41:21,058 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b224c7c31e6ff664ce67ac884f573d7, ASSIGN 2024-11-25T03:41:21,058 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b224c7c31e6ff664ce67ac884f573d7, server=d73269e94d72,33145,1732506072477 2024-11-25T03:41:21,058 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477 2024-11-25T03:41:21,058 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8b5ae4b02f22b24f40505bb660957004, server=d73269e94d72,38397,1732506072528 2024-11-25T03:41:21,058 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,058 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,058 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,058 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,058 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,058 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,058 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,059 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,059 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN 2024-11-25T03:41:21,059 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,059 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN 2024-11-25T03:41:21,059 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,059 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN 2024-11-25T03:41:21,059 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4705fa4ae38907dd8b9783be2dff29da, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,059 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4705fa4ae38907dd8b9783be2dff29da, ASSIGN 2024-11-25T03:41:21,059 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c769ca025a015554efcc98619008be96, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,059 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c769ca025a015554efcc98619008be96, ASSIGN 2024-11-25T03:41:21,059 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b5ae4b02f22b24f40505bb660957004, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,059 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b5ae4b02f22b24f40505bb660957004, ASSIGN 2024-11-25T03:41:21,059 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f64cb2042c701322ac00e312f65f16, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,059 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f64cb2042c701322ac00e312f65f16, ASSIGN 2024-11-25T03:41:21,059 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b224c7c31e6ff664ce67ac884f573d7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,059 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b224c7c31e6ff664ce67ac884f573d7, ASSIGN 2024-11-25T03:41:21,059 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b224c7c31e6ff664ce67ac884f573d7, server=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,059 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,059 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8b5ae4b02f22b24f40505bb660957004, server=d73269e94d72,38397,1732506072528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,060 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 11 msec 2024-11-25T03:41:21,061 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=65 2024-11-25T03:41:21,061 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=d73269e94d72,33145,1732506072477, table=hbase:meta, region=1588230740 2024-11-25T03:41:21,061 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d73269e94d72,33145,1732506072477, state=OPEN 2024-11-25T03:41:21,071 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38397-0x1017014aa590003, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:21,071 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40325-0x1017014aa590000, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:21,071 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33145-0x1017014aa590002, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:21,071 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:21,071 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:21,071 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:21,072 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN to state=OFFLINE, location=null, table=test, region=857a6b6999e909a9b73e698c69291205 to restore RIT 2024-11-25T03:41:21,072 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN to state=OFFLINE, location=null, table=test, region=3b442507eaac3b5222a3c4f97ff80208 to restore RIT 2024-11-25T03:41:21,072 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN to state=OFFLINE, location=null, table=test, region=922ebf14d68aba40da3a2e99777c777b to restore RIT 2024-11-25T03:41:21,072 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4705fa4ae38907dd8b9783be2dff29da, ASSIGN to state=OFFLINE, location=null, table=test, region=4705fa4ae38907dd8b9783be2dff29da to restore RIT 2024-11-25T03:41:21,072 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c769ca025a015554efcc98619008be96, ASSIGN to state=OFFLINE, location=null, table=test, region=c769ca025a015554efcc98619008be96 to restore RIT 2024-11-25T03:41:21,072 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b5ae4b02f22b24f40505bb660957004, ASSIGN to state=OFFLINE, location=null, table=test, region=8b5ae4b02f22b24f40505bb660957004 to restore RIT 2024-11-25T03:41:21,072 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f64cb2042c701322ac00e312f65f16, ASSIGN to state=OFFLINE, location=null, table=test, region=f5f64cb2042c701322ac00e312f65f16 to restore RIT 2024-11-25T03:41:21,072 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b224c7c31e6ff664ce67ac884f573d7, ASSIGN to state=OFFLINE, location=null, table=test, region=2b224c7c31e6ff664ce67ac884f573d7 to restore RIT 2024-11-25T03:41:21,073 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732506111073 2024-11-25T03:41:21,073 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-25T03:41:21,073 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T03:41:21,074 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-25T03:41:21,075 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-25T03:41:21,075 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=76, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477 2024-11-25T03:41:21,075 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T03:41:21,075 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=76, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T03:41:21,079 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3b442507eaac3b5222a3c4f97ff80208, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,079 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c73ee0e3b504288792edda3ec9a26049, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,080 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e3a490d70b339ac73875ea7638f934f, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,080 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85a5fab26752f98e6966cd407e31db69, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,080 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a7d4164efdaed87f07b6f76a94819b3, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,080 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bba1924bf10ad03aeb55628aeffc2363, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,080 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=857a6b6999e909a9b73e698c69291205, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,080 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e791add66a4fee50be34aa83a6d502a, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,081 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c0b379dd00b53bde73bca0e12f321ce, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,081 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5f64cb2042c701322ac00e312f65f16, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,081 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4b8682bf62ac734aa4174eba4ee1ddc, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,081 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb9ff3be6d639a626d88387efa9120fd, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,081 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0c260efb5b2a8a6137010d439efc8d47, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,082 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4705fa4ae38907dd8b9783be2dff29da, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,082 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5da71c9c32aa33cf6f067ea6d82ae84f, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,082 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fc93e882c2b7c5acdb53e9cf1071dc8, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,082 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=922ebf14d68aba40da3a2e99777c777b, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,082 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=745356a682ec14d9f6e2066d456cab06, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,082 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0698446036b7ae35179ebc9b07274db5, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b5ae4b02f22b24f40505bb660957004, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6aed1980a8b63b44c776d0990f923aa7, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=706facb174cc9547b71f08aa672ddcee, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b224c7c31e6ff664ce67ac884f573d7, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f812dacea03120e7730979dc37e38c6, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df71ad9490ccea77dfed73e590672db8, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,084 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c769ca025a015554efcc98619008be96, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,084 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-25T03:41:21,084 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732506141084 2024-11-25T03:41:21,084 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732506201084 2024-11-25T03:41:21,084 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 11 msec 2024-11-25T03:41:21,084 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-25T03:41:21,084 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T03:41:21,084 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-25T03:41:21,084 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-25T03:41:21,084 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-25T03:41:21,084 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-25T03:41:21,085 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-25T03:41:21,085 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-25T03:41:21,086 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-25T03:41:21,088 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-25T03:41:21,089 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-25T03:41:21,089 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-25T03:41:21,089 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,089 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,089 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-25T03:41:21,089 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,089 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,089 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,089 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,089 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,089 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,089 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-25T03:41:21,089 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-25T03:41:21,089 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-25T03:41:21,089 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,089 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-25T03:41:21,090 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-25T03:41:21,091 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-25T03:41:21,091 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, REOPEN/MOVE 2024-11-25T03:41:21,093 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true 2024-11-25T03:41:21,093 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 2024-11-25T03:41:21,093 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 2024-11-25T03:41:21,093 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T03:41:21,093 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d73269e94d72,33145,1732506072477 2024-11-25T03:41:21,093 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 2024-11-25T03:41:21,093 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 2024-11-25T03:41:21,093 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN 2024-11-25T03:41:21,093 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN 2024-11-25T03:41:21,093 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN 2024-11-25T03:41:21,093 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4705fa4ae38907dd8b9783be2dff29da, ASSIGN 2024-11-25T03:41:21,093 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c769ca025a015554efcc98619008be96, ASSIGN 2024-11-25T03:41:21,093 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b5ae4b02f22b24f40505bb660957004, ASSIGN 2024-11-25T03:41:21,093 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f64cb2042c701322ac00e312f65f16, ASSIGN 2024-11-25T03:41:21,093 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b224c7c31e6ff664ce67ac884f573d7, ASSIGN 2024-11-25T03:41:21,093 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b224c7c31e6ff664ce67ac884f573d7, server=d73269e94d72,33145,1732506072477 2024-11-25T03:41:21,093 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477 2024-11-25T03:41:21,094 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,094 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,094 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,094 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,094 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,094 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,094 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,094 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,094 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN 2024-11-25T03:41:21,094 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,094 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN 2024-11-25T03:41:21,094 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,094 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN 2024-11-25T03:41:21,094 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4705fa4ae38907dd8b9783be2dff29da, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,094 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4705fa4ae38907dd8b9783be2dff29da, ASSIGN 2024-11-25T03:41:21,094 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c769ca025a015554efcc98619008be96, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,094 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c769ca025a015554efcc98619008be96, ASSIGN 2024-11-25T03:41:21,094 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b5ae4b02f22b24f40505bb660957004, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,094 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b5ae4b02f22b24f40505bb660957004, ASSIGN 2024-11-25T03:41:21,094 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f64cb2042c701322ac00e312f65f16, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,094 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f64cb2042c701322ac00e312f65f16, ASSIGN 2024-11-25T03:41:21,095 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b224c7c31e6ff664ce67ac884f573d7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,095 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b224c7c31e6ff664ce67ac884f573d7, ASSIGN 2024-11-25T03:41:21,095 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b224c7c31e6ff664ce67ac884f573d7, server=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,095 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,095 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-25T03:41:21,096 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=65 2024-11-25T03:41:21,096 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=d73269e94d72,33145,1732506072477, table=hbase:meta, region=1588230740 2024-11-25T03:41:21,096 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d73269e94d72,33145,1732506072477, state=OPEN 2024-11-25T03:41:21,103 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33145-0x1017014aa590002, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:21,103 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38397-0x1017014aa590003, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:21,103 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40325-0x1017014aa590000, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:21,103 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:21,103 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:21,103 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:21,104 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN to state=OFFLINE, location=null, table=test, region=857a6b6999e909a9b73e698c69291205 to restore RIT 2024-11-25T03:41:21,104 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN to state=OFFLINE, location=null, table=test, region=3b442507eaac3b5222a3c4f97ff80208 to restore RIT 2024-11-25T03:41:21,104 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN to state=OFFLINE, location=null, table=test, region=922ebf14d68aba40da3a2e99777c777b to restore RIT 2024-11-25T03:41:21,104 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4705fa4ae38907dd8b9783be2dff29da, ASSIGN to state=OFFLINE, location=null, table=test, region=4705fa4ae38907dd8b9783be2dff29da to restore RIT 2024-11-25T03:41:21,104 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c769ca025a015554efcc98619008be96, ASSIGN to state=OFFLINE, location=null, table=test, region=c769ca025a015554efcc98619008be96 to restore RIT 2024-11-25T03:41:21,104 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b5ae4b02f22b24f40505bb660957004, ASSIGN to state=OFFLINE, location=null, table=test, region=8b5ae4b02f22b24f40505bb660957004 to restore RIT 2024-11-25T03:41:21,104 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f64cb2042c701322ac00e312f65f16, ASSIGN to state=OFFLINE, location=null, table=test, region=f5f64cb2042c701322ac00e312f65f16 to restore RIT 2024-11-25T03:41:21,104 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b224c7c31e6ff664ce67ac884f573d7, ASSIGN to state=OFFLINE, location=null, table=test, region=2b224c7c31e6ff664ce67ac884f573d7 to restore RIT 2024-11-25T03:41:21,105 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732506111105 2024-11-25T03:41:21,105 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-25T03:41:21,105 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T03:41:21,106 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-25T03:41:21,107 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-25T03:41:21,107 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b224c7c31e6ff664ce67ac884f573d7, server=d73269e94d72,33145,1732506072477 2024-11-25T03:41:21,107 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T03:41:21,107 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b224c7c31e6ff664ce67ac884f573d7, server=d73269e94d72,33145,1732506072477 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T03:41:21,111 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3b442507eaac3b5222a3c4f97ff80208, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,112 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c73ee0e3b504288792edda3ec9a26049, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,112 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e3a490d70b339ac73875ea7638f934f, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,112 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85a5fab26752f98e6966cd407e31db69, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,112 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a7d4164efdaed87f07b6f76a94819b3, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,112 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bba1924bf10ad03aeb55628aeffc2363, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,112 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=857a6b6999e909a9b73e698c69291205, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,112 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e791add66a4fee50be34aa83a6d502a, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,112 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c0b379dd00b53bde73bca0e12f321ce, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,113 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5f64cb2042c701322ac00e312f65f16, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,113 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4b8682bf62ac734aa4174eba4ee1ddc, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,113 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb9ff3be6d639a626d88387efa9120fd, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,113 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0c260efb5b2a8a6137010d439efc8d47, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,113 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4705fa4ae38907dd8b9783be2dff29da, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,113 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5da71c9c32aa33cf6f067ea6d82ae84f, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,113 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fc93e882c2b7c5acdb53e9cf1071dc8, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,113 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=922ebf14d68aba40da3a2e99777c777b, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,114 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=745356a682ec14d9f6e2066d456cab06, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,114 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0698446036b7ae35179ebc9b07274db5, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,114 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b5ae4b02f22b24f40505bb660957004, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,114 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6aed1980a8b63b44c776d0990f923aa7, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,114 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=706facb174cc9547b71f08aa672ddcee, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,114 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b224c7c31e6ff664ce67ac884f573d7, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,114 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f812dacea03120e7730979dc37e38c6, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,114 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df71ad9490ccea77dfed73e590672db8, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,114 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c769ca025a015554efcc98619008be96, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,115 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-25T03:41:21,115 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732506141115 2024-11-25T03:41:21,115 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732506201115 2024-11-25T03:41:21,115 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-25T03:41:21,115 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-25T03:41:21,115 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T03:41:21,115 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-25T03:41:21,115 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-25T03:41:21,115 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-25T03:41:21,115 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-25T03:41:21,115 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-25T03:41:21,115 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-25T03:41:21,116 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-25T03:41:21,118 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-25T03:41:21,118 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-25T03:41:21,118 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-25T03:41:21,118 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,118 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,118 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,118 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,118 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,118 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,118 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,118 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,119 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-25T03:41:21,119 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-25T03:41:21,119 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-25T03:41:21,119 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,119 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-25T03:41:21,119 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-25T03:41:21,120 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-25T03:41:21,120 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, REOPEN/MOVE 2024-11-25T03:41:21,121 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true 2024-11-25T03:41:21,121 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 2024-11-25T03:41:21,122 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 2024-11-25T03:41:21,122 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T03:41:21,122 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d73269e94d72,33145,1732506072477 2024-11-25T03:41:21,122 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 2024-11-25T03:41:21,122 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 2024-11-25T03:41:21,122 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN 2024-11-25T03:41:21,122 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN 2024-11-25T03:41:21,122 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN 2024-11-25T03:41:21,122 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4705fa4ae38907dd8b9783be2dff29da, ASSIGN 2024-11-25T03:41:21,122 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c769ca025a015554efcc98619008be96, ASSIGN 2024-11-25T03:41:21,122 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b5ae4b02f22b24f40505bb660957004, ASSIGN 2024-11-25T03:41:21,122 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f64cb2042c701322ac00e312f65f16, ASSIGN 2024-11-25T03:41:21,122 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b224c7c31e6ff664ce67ac884f573d7, ASSIGN 2024-11-25T03:41:21,122 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b224c7c31e6ff664ce67ac884f573d7, server=d73269e94d72,33145,1732506072477 2024-11-25T03:41:21,122 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,122 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,122 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,122 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,122 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,122 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,122 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,122 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,122 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN 2024-11-25T03:41:21,122 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,122 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN 2024-11-25T03:41:21,122 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,122 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN 2024-11-25T03:41:21,122 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4705fa4ae38907dd8b9783be2dff29da, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,122 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4705fa4ae38907dd8b9783be2dff29da, ASSIGN 2024-11-25T03:41:21,123 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c769ca025a015554efcc98619008be96, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,123 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c769ca025a015554efcc98619008be96, ASSIGN 2024-11-25T03:41:21,123 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b5ae4b02f22b24f40505bb660957004, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,123 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b5ae4b02f22b24f40505bb660957004, ASSIGN 2024-11-25T03:41:21,123 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f64cb2042c701322ac00e312f65f16, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,123 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f64cb2042c701322ac00e312f65f16, ASSIGN 2024-11-25T03:41:21,123 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b224c7c31e6ff664ce67ac884f573d7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,123 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b224c7c31e6ff664ce67ac884f573d7, ASSIGN 2024-11-25T03:41:21,123 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b224c7c31e6ff664ce67ac884f573d7, server=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,123 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-25T03:41:21,124 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=65 2024-11-25T03:41:21,124 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=d73269e94d72,33145,1732506072477, table=hbase:meta, region=1588230740 2024-11-25T03:41:21,124 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d73269e94d72,33145,1732506072477, state=OPEN 2024-11-25T03:41:21,131 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40325 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T03:41:21,132 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40325 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T03:41:21,134 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33145-0x1017014aa590002, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:21,134 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38397-0x1017014aa590003, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:21,134 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40325-0x1017014aa590000, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:21,134 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:21,134 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:21,134 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:21,135 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN to state=OFFLINE, location=null, table=test, region=857a6b6999e909a9b73e698c69291205 to restore RIT 2024-11-25T03:41:21,135 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN to state=OFFLINE, location=null, table=test, region=3b442507eaac3b5222a3c4f97ff80208 to restore RIT 2024-11-25T03:41:21,135 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN to state=OFFLINE, location=null, table=test, region=922ebf14d68aba40da3a2e99777c777b to restore RIT 2024-11-25T03:41:21,135 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4705fa4ae38907dd8b9783be2dff29da, ASSIGN to state=OFFLINE, location=null, table=test, region=4705fa4ae38907dd8b9783be2dff29da to restore RIT 2024-11-25T03:41:21,135 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c769ca025a015554efcc98619008be96, ASSIGN to state=OFFLINE, location=null, table=test, region=c769ca025a015554efcc98619008be96 to restore RIT 2024-11-25T03:41:21,135 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b5ae4b02f22b24f40505bb660957004, ASSIGN to state=OFFLINE, location=null, table=test, region=8b5ae4b02f22b24f40505bb660957004 to restore RIT 2024-11-25T03:41:21,135 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f64cb2042c701322ac00e312f65f16, ASSIGN to state=OFFLINE, location=null, table=test, region=f5f64cb2042c701322ac00e312f65f16 to restore RIT 2024-11-25T03:41:21,136 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b224c7c31e6ff664ce67ac884f573d7, ASSIGN to state=OFFLINE, location=null, table=test, region=2b224c7c31e6ff664ce67ac884f573d7 to restore RIT 2024-11-25T03:41:21,136 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732506111136 2024-11-25T03:41:21,136 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-25T03:41:21,136 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T03:41:21,137 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-25T03:41:21,138 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-25T03:41:21,138 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b224c7c31e6ff664ce67ac884f573d7, ASSIGN 2024-11-25T03:41:21,138 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T03:41:21,138 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b224c7c31e6ff664ce67ac884f573d7, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T03:41:21,141 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3b442507eaac3b5222a3c4f97ff80208, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,141 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c73ee0e3b504288792edda3ec9a26049, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,142 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e3a490d70b339ac73875ea7638f934f, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,142 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85a5fab26752f98e6966cd407e31db69, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,142 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a7d4164efdaed87f07b6f76a94819b3, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,142 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bba1924bf10ad03aeb55628aeffc2363, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,142 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=857a6b6999e909a9b73e698c69291205, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,142 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e791add66a4fee50be34aa83a6d502a, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,142 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c0b379dd00b53bde73bca0e12f321ce, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,142 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5f64cb2042c701322ac00e312f65f16, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,143 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4b8682bf62ac734aa4174eba4ee1ddc, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,143 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb9ff3be6d639a626d88387efa9120fd, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,143 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0c260efb5b2a8a6137010d439efc8d47, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,143 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4705fa4ae38907dd8b9783be2dff29da, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,143 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5da71c9c32aa33cf6f067ea6d82ae84f, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,143 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fc93e882c2b7c5acdb53e9cf1071dc8, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,143 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=922ebf14d68aba40da3a2e99777c777b, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,143 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=745356a682ec14d9f6e2066d456cab06, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,144 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0698446036b7ae35179ebc9b07274db5, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,144 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b5ae4b02f22b24f40505bb660957004, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,144 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6aed1980a8b63b44c776d0990f923aa7, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,144 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=706facb174cc9547b71f08aa672ddcee, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,144 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b224c7c31e6ff664ce67ac884f573d7, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,144 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f812dacea03120e7730979dc37e38c6, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,144 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df71ad9490ccea77dfed73e590672db8, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,144 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c769ca025a015554efcc98619008be96, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,144 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-25T03:41:21,144 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732506141144 2024-11-25T03:41:21,145 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732506201145 2024-11-25T03:41:21,145 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-25T03:41:21,145 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-25T03:41:21,145 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T03:41:21,145 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-25T03:41:21,145 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-25T03:41:21,145 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-25T03:41:21,145 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-25T03:41:21,145 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-25T03:41:21,145 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-25T03:41:21,146 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-25T03:41:21,148 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-25T03:41:21,148 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-25T03:41:21,148 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-25T03:41:21,148 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,148 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,148 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,148 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,148 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,148 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,148 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,149 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,149 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-25T03:41:21,149 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-25T03:41:21,149 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,149 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-25T03:41:21,149 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-25T03:41:21,150 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-25T03:41:21,150 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, REOPEN/MOVE 2024-11-25T03:41:21,152 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true 2024-11-25T03:41:21,152 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 2024-11-25T03:41:21,152 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 2024-11-25T03:41:21,152 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T03:41:21,152 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d73269e94d72,33145,1732506072477 2024-11-25T03:41:21,152 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 2024-11-25T03:41:21,152 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 2024-11-25T03:41:21,152 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN 2024-11-25T03:41:21,152 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN 2024-11-25T03:41:21,152 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN 2024-11-25T03:41:21,152 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4705fa4ae38907dd8b9783be2dff29da, ASSIGN 2024-11-25T03:41:21,152 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c769ca025a015554efcc98619008be96, ASSIGN 2024-11-25T03:41:21,152 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b5ae4b02f22b24f40505bb660957004, ASSIGN 2024-11-25T03:41:21,152 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f64cb2042c701322ac00e312f65f16, ASSIGN 2024-11-25T03:41:21,153 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b224c7c31e6ff664ce67ac884f573d7, ASSIGN 2024-11-25T03:41:21,153 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,153 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,153 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,153 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,153 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,153 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,153 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,153 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,153 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN 2024-11-25T03:41:21,153 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,153 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN 2024-11-25T03:41:21,153 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,153 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN 2024-11-25T03:41:21,153 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4705fa4ae38907dd8b9783be2dff29da, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,153 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4705fa4ae38907dd8b9783be2dff29da, ASSIGN 2024-11-25T03:41:21,153 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c769ca025a015554efcc98619008be96, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,154 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c769ca025a015554efcc98619008be96, ASSIGN 2024-11-25T03:41:21,154 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b5ae4b02f22b24f40505bb660957004, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,154 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b5ae4b02f22b24f40505bb660957004, ASSIGN 2024-11-25T03:41:21,154 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f64cb2042c701322ac00e312f65f16, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,154 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f64cb2042c701322ac00e312f65f16, ASSIGN 2024-11-25T03:41:21,154 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b224c7c31e6ff664ce67ac884f573d7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,154 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b224c7c31e6ff664ce67ac884f573d7, ASSIGN 2024-11-25T03:41:21,154 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-25T03:41:21,155 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=65 2024-11-25T03:41:21,155 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=d73269e94d72,33145,1732506072477, table=hbase:meta, region=1588230740 2024-11-25T03:41:21,156 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d73269e94d72,33145,1732506072477, state=OPEN 2024-11-25T03:41:21,166 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33145-0x1017014aa590002, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:21,166 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40325-0x1017014aa590000, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:21,166 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38397-0x1017014aa590003, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:21,166 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:21,166 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:21,166 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:21,167 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN to state=OFFLINE, location=null, table=test, region=857a6b6999e909a9b73e698c69291205 to restore RIT 2024-11-25T03:41:21,167 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN to state=OFFLINE, location=null, table=test, region=3b442507eaac3b5222a3c4f97ff80208 to restore RIT 2024-11-25T03:41:21,167 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN to state=OFFLINE, location=null, table=test, region=922ebf14d68aba40da3a2e99777c777b to restore RIT 2024-11-25T03:41:21,167 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4705fa4ae38907dd8b9783be2dff29da, ASSIGN to state=OFFLINE, location=null, table=test, region=4705fa4ae38907dd8b9783be2dff29da to restore RIT 2024-11-25T03:41:21,167 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c769ca025a015554efcc98619008be96, ASSIGN to state=OFFLINE, location=null, table=test, region=c769ca025a015554efcc98619008be96 to restore RIT 2024-11-25T03:41:21,167 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b5ae4b02f22b24f40505bb660957004, ASSIGN to state=OFFLINE, location=null, table=test, region=8b5ae4b02f22b24f40505bb660957004 to restore RIT 2024-11-25T03:41:21,167 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f64cb2042c701322ac00e312f65f16, ASSIGN to state=OFFLINE, location=null, table=test, region=f5f64cb2042c701322ac00e312f65f16 to restore RIT 2024-11-25T03:41:21,167 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b224c7c31e6ff664ce67ac884f573d7, ASSIGN to state=OFFLINE, location=null, table=test, region=2b224c7c31e6ff664ce67ac884f573d7 to restore RIT 2024-11-25T03:41:21,168 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732506111168 2024-11-25T03:41:21,168 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-25T03:41:21,168 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T03:41:21,169 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-25T03:41:21,170 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-25T03:41:21,170 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f64cb2042c701322ac00e312f65f16, ASSIGN 2024-11-25T03:41:21,170 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T03:41:21,170 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f64cb2042c701322ac00e312f65f16, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T03:41:21,173 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3b442507eaac3b5222a3c4f97ff80208, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,173 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c73ee0e3b504288792edda3ec9a26049, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,174 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e3a490d70b339ac73875ea7638f934f, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,174 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85a5fab26752f98e6966cd407e31db69, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,174 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a7d4164efdaed87f07b6f76a94819b3, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,174 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bba1924bf10ad03aeb55628aeffc2363, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,174 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=857a6b6999e909a9b73e698c69291205, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,174 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e791add66a4fee50be34aa83a6d502a, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,175 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c0b379dd00b53bde73bca0e12f321ce, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,175 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5f64cb2042c701322ac00e312f65f16, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,175 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4b8682bf62ac734aa4174eba4ee1ddc, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,175 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb9ff3be6d639a626d88387efa9120fd, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,175 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0c260efb5b2a8a6137010d439efc8d47, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,175 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4705fa4ae38907dd8b9783be2dff29da, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,175 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5da71c9c32aa33cf6f067ea6d82ae84f, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,176 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fc93e882c2b7c5acdb53e9cf1071dc8, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,176 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=922ebf14d68aba40da3a2e99777c777b, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,176 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=745356a682ec14d9f6e2066d456cab06, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,176 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0698446036b7ae35179ebc9b07274db5, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,176 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b5ae4b02f22b24f40505bb660957004, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,176 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6aed1980a8b63b44c776d0990f923aa7, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,177 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=706facb174cc9547b71f08aa672ddcee, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,177 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b224c7c31e6ff664ce67ac884f573d7, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,177 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f812dacea03120e7730979dc37e38c6, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,177 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df71ad9490ccea77dfed73e590672db8, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,177 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c769ca025a015554efcc98619008be96, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,177 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-25T03:41:21,177 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732506141177 2024-11-25T03:41:21,177 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732506201177 2024-11-25T03:41:21,177 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-25T03:41:21,178 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-25T03:41:21,178 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T03:41:21,178 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-25T03:41:21,178 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-25T03:41:21,178 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-25T03:41:21,178 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-25T03:41:21,178 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-25T03:41:21,178 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-25T03:41:21,179 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-25T03:41:21,182 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-25T03:41:21,182 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-25T03:41:21,182 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-25T03:41:21,182 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,182 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,182 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,182 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,182 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,182 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,182 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,182 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-25T03:41:21,182 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-25T03:41:21,183 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,183 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-25T03:41:21,183 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-25T03:41:21,184 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-25T03:41:21,184 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, REOPEN/MOVE 2024-11-25T03:41:21,185 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true 2024-11-25T03:41:21,185 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 2024-11-25T03:41:21,185 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 2024-11-25T03:41:21,185 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T03:41:21,185 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d73269e94d72,33145,1732506072477 2024-11-25T03:41:21,185 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 2024-11-25T03:41:21,185 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 2024-11-25T03:41:21,186 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN 2024-11-25T03:41:21,186 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN 2024-11-25T03:41:21,186 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN 2024-11-25T03:41:21,186 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4705fa4ae38907dd8b9783be2dff29da, ASSIGN 2024-11-25T03:41:21,186 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c769ca025a015554efcc98619008be96, ASSIGN 2024-11-25T03:41:21,186 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b5ae4b02f22b24f40505bb660957004, ASSIGN 2024-11-25T03:41:21,186 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f64cb2042c701322ac00e312f65f16, ASSIGN 2024-11-25T03:41:21,186 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,186 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,186 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,186 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,186 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,186 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,186 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,186 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,186 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN 2024-11-25T03:41:21,186 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,186 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN 2024-11-25T03:41:21,186 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,186 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN 2024-11-25T03:41:21,187 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4705fa4ae38907dd8b9783be2dff29da, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,187 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4705fa4ae38907dd8b9783be2dff29da, ASSIGN 2024-11-25T03:41:21,187 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c769ca025a015554efcc98619008be96, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,187 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c769ca025a015554efcc98619008be96, ASSIGN 2024-11-25T03:41:21,187 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b5ae4b02f22b24f40505bb660957004, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,187 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b5ae4b02f22b24f40505bb660957004, ASSIGN 2024-11-25T03:41:21,187 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f64cb2042c701322ac00e312f65f16, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,187 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f64cb2042c701322ac00e312f65f16, ASSIGN 2024-11-25T03:41:21,187 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-25T03:41:21,188 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=65 2024-11-25T03:41:21,188 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=d73269e94d72,33145,1732506072477, table=hbase:meta, region=1588230740 2024-11-25T03:41:21,188 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d73269e94d72,33145,1732506072477, state=OPEN 2024-11-25T03:41:21,197 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40325-0x1017014aa590000, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:21,197 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33145-0x1017014aa590002, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:21,197 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38397-0x1017014aa590003, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:21,198 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:21,198 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:21,198 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:21,198 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN to state=OFFLINE, location=null, table=test, region=857a6b6999e909a9b73e698c69291205 to restore RIT 2024-11-25T03:41:21,198 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN to state=OFFLINE, location=null, table=test, region=3b442507eaac3b5222a3c4f97ff80208 to restore RIT 2024-11-25T03:41:21,198 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN to state=OFFLINE, location=null, table=test, region=922ebf14d68aba40da3a2e99777c777b to restore RIT 2024-11-25T03:41:21,198 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4705fa4ae38907dd8b9783be2dff29da, ASSIGN to state=OFFLINE, location=null, table=test, region=4705fa4ae38907dd8b9783be2dff29da to restore RIT 2024-11-25T03:41:21,198 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c769ca025a015554efcc98619008be96, ASSIGN to state=OFFLINE, location=null, table=test, region=c769ca025a015554efcc98619008be96 to restore RIT 2024-11-25T03:41:21,198 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b5ae4b02f22b24f40505bb660957004, ASSIGN to state=OFFLINE, location=null, table=test, region=8b5ae4b02f22b24f40505bb660957004 to restore RIT 2024-11-25T03:41:21,199 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f64cb2042c701322ac00e312f65f16, ASSIGN to state=OFFLINE, location=null, table=test, region=f5f64cb2042c701322ac00e312f65f16 to restore RIT 2024-11-25T03:41:21,199 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732506111199 2024-11-25T03:41:21,199 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-25T03:41:21,199 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T03:41:21,200 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-25T03:41:21,202 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-25T03:41:21,202 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b5ae4b02f22b24f40505bb660957004, ASSIGN 2024-11-25T03:41:21,202 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T03:41:21,202 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b5ae4b02f22b24f40505bb660957004, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T03:41:21,205 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3b442507eaac3b5222a3c4f97ff80208, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,205 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c73ee0e3b504288792edda3ec9a26049, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,205 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e3a490d70b339ac73875ea7638f934f, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,205 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85a5fab26752f98e6966cd407e31db69, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,205 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a7d4164efdaed87f07b6f76a94819b3, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,206 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bba1924bf10ad03aeb55628aeffc2363, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,206 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=857a6b6999e909a9b73e698c69291205, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,206 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e791add66a4fee50be34aa83a6d502a, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,206 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c0b379dd00b53bde73bca0e12f321ce, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,206 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5f64cb2042c701322ac00e312f65f16, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,206 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4b8682bf62ac734aa4174eba4ee1ddc, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,206 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb9ff3be6d639a626d88387efa9120fd, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,207 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0c260efb5b2a8a6137010d439efc8d47, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,207 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4705fa4ae38907dd8b9783be2dff29da, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,207 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5da71c9c32aa33cf6f067ea6d82ae84f, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,207 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fc93e882c2b7c5acdb53e9cf1071dc8, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,207 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=922ebf14d68aba40da3a2e99777c777b, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,207 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=745356a682ec14d9f6e2066d456cab06, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,207 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0698446036b7ae35179ebc9b07274db5, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,208 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b5ae4b02f22b24f40505bb660957004, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,208 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6aed1980a8b63b44c776d0990f923aa7, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,208 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=706facb174cc9547b71f08aa672ddcee, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,208 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b224c7c31e6ff664ce67ac884f573d7, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,208 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f812dacea03120e7730979dc37e38c6, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,208 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df71ad9490ccea77dfed73e590672db8, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,209 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c769ca025a015554efcc98619008be96, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,209 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-25T03:41:21,209 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732506141209 2024-11-25T03:41:21,209 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732506201209 2024-11-25T03:41:21,209 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-25T03:41:21,209 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-25T03:41:21,209 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T03:41:21,209 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-25T03:41:21,209 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-25T03:41:21,210 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-25T03:41:21,210 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-25T03:41:21,210 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-25T03:41:21,210 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-25T03:41:21,211 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-25T03:41:21,213 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-25T03:41:21,213 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-25T03:41:21,213 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-25T03:41:21,214 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,214 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,214 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,214 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,214 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,214 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,214 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-25T03:41:21,214 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-25T03:41:21,214 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,214 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-25T03:41:21,214 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-25T03:41:21,215 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-25T03:41:21,215 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, REOPEN/MOVE 2024-11-25T03:41:21,216 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true 2024-11-25T03:41:21,216 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 2024-11-25T03:41:21,217 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 2024-11-25T03:41:21,217 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T03:41:21,217 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d73269e94d72,33145,1732506072477 2024-11-25T03:41:21,217 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 2024-11-25T03:41:21,217 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 2024-11-25T03:41:21,217 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN 2024-11-25T03:41:21,217 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN 2024-11-25T03:41:21,217 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN 2024-11-25T03:41:21,217 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4705fa4ae38907dd8b9783be2dff29da, ASSIGN 2024-11-25T03:41:21,217 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c769ca025a015554efcc98619008be96, ASSIGN 2024-11-25T03:41:21,217 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b5ae4b02f22b24f40505bb660957004, ASSIGN 2024-11-25T03:41:21,217 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,217 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,217 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,217 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,217 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,217 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,217 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,217 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,217 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN 2024-11-25T03:41:21,217 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,217 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN 2024-11-25T03:41:21,217 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,217 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN 2024-11-25T03:41:21,217 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4705fa4ae38907dd8b9783be2dff29da, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,217 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4705fa4ae38907dd8b9783be2dff29da, ASSIGN 2024-11-25T03:41:21,217 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c769ca025a015554efcc98619008be96, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,217 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c769ca025a015554efcc98619008be96, ASSIGN 2024-11-25T03:41:21,218 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b5ae4b02f22b24f40505bb660957004, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,218 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b5ae4b02f22b24f40505bb660957004, ASSIGN 2024-11-25T03:41:21,218 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-25T03:41:21,219 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=65 2024-11-25T03:41:21,219 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=d73269e94d72,33145,1732506072477, table=hbase:meta, region=1588230740 2024-11-25T03:41:21,219 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d73269e94d72,33145,1732506072477, state=OPEN 2024-11-25T03:41:21,229 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40325-0x1017014aa590000, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:21,229 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38397-0x1017014aa590003, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:21,229 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33145-0x1017014aa590002, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:21,229 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:21,229 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:21,229 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:21,230 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN to state=OFFLINE, location=null, table=test, region=857a6b6999e909a9b73e698c69291205 to restore RIT 2024-11-25T03:41:21,230 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN to state=OFFLINE, location=null, table=test, region=3b442507eaac3b5222a3c4f97ff80208 to restore RIT 2024-11-25T03:41:21,230 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN to state=OFFLINE, location=null, table=test, region=922ebf14d68aba40da3a2e99777c777b to restore RIT 2024-11-25T03:41:21,230 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4705fa4ae38907dd8b9783be2dff29da, ASSIGN to state=OFFLINE, location=null, table=test, region=4705fa4ae38907dd8b9783be2dff29da to restore RIT 2024-11-25T03:41:21,230 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c769ca025a015554efcc98619008be96, ASSIGN to state=OFFLINE, location=null, table=test, region=c769ca025a015554efcc98619008be96 to restore RIT 2024-11-25T03:41:21,230 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b5ae4b02f22b24f40505bb660957004, ASSIGN to state=OFFLINE, location=null, table=test, region=8b5ae4b02f22b24f40505bb660957004 to restore RIT 2024-11-25T03:41:21,231 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732506111231 2024-11-25T03:41:21,231 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-25T03:41:21,231 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T03:41:21,232 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-25T03:41:21,233 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-25T03:41:21,233 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c769ca025a015554efcc98619008be96, ASSIGN 2024-11-25T03:41:21,233 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T03:41:21,234 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c769ca025a015554efcc98619008be96, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T03:41:21,239 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3b442507eaac3b5222a3c4f97ff80208, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,240 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c73ee0e3b504288792edda3ec9a26049, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,240 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e3a490d70b339ac73875ea7638f934f, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,240 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85a5fab26752f98e6966cd407e31db69, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,240 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a7d4164efdaed87f07b6f76a94819b3, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,240 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bba1924bf10ad03aeb55628aeffc2363, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,240 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=857a6b6999e909a9b73e698c69291205, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,240 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e791add66a4fee50be34aa83a6d502a, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,241 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c0b379dd00b53bde73bca0e12f321ce, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,241 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5f64cb2042c701322ac00e312f65f16, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,241 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4b8682bf62ac734aa4174eba4ee1ddc, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,241 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb9ff3be6d639a626d88387efa9120fd, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,241 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0c260efb5b2a8a6137010d439efc8d47, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,241 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4705fa4ae38907dd8b9783be2dff29da, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,241 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5da71c9c32aa33cf6f067ea6d82ae84f, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,241 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fc93e882c2b7c5acdb53e9cf1071dc8, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,242 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=922ebf14d68aba40da3a2e99777c777b, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,242 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=745356a682ec14d9f6e2066d456cab06, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,242 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0698446036b7ae35179ebc9b07274db5, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,242 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b5ae4b02f22b24f40505bb660957004, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,242 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6aed1980a8b63b44c776d0990f923aa7, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,242 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=706facb174cc9547b71f08aa672ddcee, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,242 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b224c7c31e6ff664ce67ac884f573d7, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,242 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f812dacea03120e7730979dc37e38c6, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,243 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df71ad9490ccea77dfed73e590672db8, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,243 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c769ca025a015554efcc98619008be96, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,243 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-25T03:41:21,243 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732506141243 2024-11-25T03:41:21,243 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732506201243 2024-11-25T03:41:21,243 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 12 msec 2024-11-25T03:41:21,243 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-25T03:41:21,243 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T03:41:21,243 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-25T03:41:21,243 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-25T03:41:21,244 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-25T03:41:21,244 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-25T03:41:21,244 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-25T03:41:21,244 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-25T03:41:21,245 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-25T03:41:21,247 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-25T03:41:21,247 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-25T03:41:21,247 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-25T03:41:21,247 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,247 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,247 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,247 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,247 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,248 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-25T03:41:21,248 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-25T03:41:21,248 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,248 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-25T03:41:21,248 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-25T03:41:21,248 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-25T03:41:21,249 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, REOPEN/MOVE 2024-11-25T03:41:21,250 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true 2024-11-25T03:41:21,250 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 2024-11-25T03:41:21,250 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 2024-11-25T03:41:21,250 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T03:41:21,250 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d73269e94d72,33145,1732506072477 2024-11-25T03:41:21,250 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 2024-11-25T03:41:21,250 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 2024-11-25T03:41:21,250 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN 2024-11-25T03:41:21,250 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN 2024-11-25T03:41:21,250 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN 2024-11-25T03:41:21,250 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4705fa4ae38907dd8b9783be2dff29da, ASSIGN 2024-11-25T03:41:21,250 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c769ca025a015554efcc98619008be96, ASSIGN 2024-11-25T03:41:21,250 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,250 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,251 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,251 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,251 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,251 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,251 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,251 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,251 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN 2024-11-25T03:41:21,251 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,251 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN 2024-11-25T03:41:21,251 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,251 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN 2024-11-25T03:41:21,251 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4705fa4ae38907dd8b9783be2dff29da, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,251 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4705fa4ae38907dd8b9783be2dff29da, ASSIGN 2024-11-25T03:41:21,251 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c769ca025a015554efcc98619008be96, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,251 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c769ca025a015554efcc98619008be96, ASSIGN 2024-11-25T03:41:21,251 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-25T03:41:21,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=65 2024-11-25T03:41:21,253 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=d73269e94d72,33145,1732506072477, table=hbase:meta, region=1588230740 2024-11-25T03:41:21,253 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d73269e94d72,33145,1732506072477, state=OPEN 2024-11-25T03:41:21,291 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38397-0x1017014aa590003, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:21,291 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33145-0x1017014aa590002, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:21,291 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40325-0x1017014aa590000, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:21,291 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:21,291 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:21,291 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:21,292 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN to state=OFFLINE, location=null, table=test, region=857a6b6999e909a9b73e698c69291205 to restore RIT 2024-11-25T03:41:21,292 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN to state=OFFLINE, location=null, table=test, region=3b442507eaac3b5222a3c4f97ff80208 to restore RIT 2024-11-25T03:41:21,292 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN to state=OFFLINE, location=null, table=test, region=922ebf14d68aba40da3a2e99777c777b to restore RIT 2024-11-25T03:41:21,292 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4705fa4ae38907dd8b9783be2dff29da, ASSIGN to state=OFFLINE, location=null, table=test, region=4705fa4ae38907dd8b9783be2dff29da to restore RIT 2024-11-25T03:41:21,292 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c769ca025a015554efcc98619008be96, ASSIGN to state=OFFLINE, location=null, table=test, region=c769ca025a015554efcc98619008be96 to restore RIT 2024-11-25T03:41:21,292 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732506111292 2024-11-25T03:41:21,292 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-25T03:41:21,292 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T03:41:21,294 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-25T03:41:21,295 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-25T03:41:21,295 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4705fa4ae38907dd8b9783be2dff29da, ASSIGN 2024-11-25T03:41:21,295 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T03:41:21,295 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4705fa4ae38907dd8b9783be2dff29da, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T03:41:21,298 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3b442507eaac3b5222a3c4f97ff80208, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,298 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c73ee0e3b504288792edda3ec9a26049, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,298 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e3a490d70b339ac73875ea7638f934f, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,298 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85a5fab26752f98e6966cd407e31db69, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,298 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a7d4164efdaed87f07b6f76a94819b3, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,299 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bba1924bf10ad03aeb55628aeffc2363, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,299 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=857a6b6999e909a9b73e698c69291205, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,299 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e791add66a4fee50be34aa83a6d502a, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,299 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c0b379dd00b53bde73bca0e12f321ce, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,299 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5f64cb2042c701322ac00e312f65f16, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,299 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4b8682bf62ac734aa4174eba4ee1ddc, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,299 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb9ff3be6d639a626d88387efa9120fd, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,299 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0c260efb5b2a8a6137010d439efc8d47, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,299 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4705fa4ae38907dd8b9783be2dff29da, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,299 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5da71c9c32aa33cf6f067ea6d82ae84f, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,299 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fc93e882c2b7c5acdb53e9cf1071dc8, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,300 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=922ebf14d68aba40da3a2e99777c777b, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,300 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=745356a682ec14d9f6e2066d456cab06, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,300 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0698446036b7ae35179ebc9b07274db5, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,300 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b5ae4b02f22b24f40505bb660957004, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,300 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6aed1980a8b63b44c776d0990f923aa7, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,300 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=706facb174cc9547b71f08aa672ddcee, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,300 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b224c7c31e6ff664ce67ac884f573d7, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,300 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f812dacea03120e7730979dc37e38c6, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,300 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df71ad9490ccea77dfed73e590672db8, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,300 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c769ca025a015554efcc98619008be96, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,300 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-25T03:41:21,301 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732506141301 2024-11-25T03:41:21,301 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732506201301 2024-11-25T03:41:21,301 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-25T03:41:21,301 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-25T03:41:21,301 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T03:41:21,301 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-25T03:41:21,301 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-25T03:41:21,301 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-25T03:41:21,301 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-25T03:41:21,301 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-25T03:41:21,302 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-25T03:41:21,302 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-25T03:41:21,304 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-25T03:41:21,304 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-25T03:41:21,304 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-25T03:41:21,304 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,304 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,304 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,304 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,304 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-25T03:41:21,304 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-25T03:41:21,304 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,304 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-25T03:41:21,304 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-25T03:41:21,305 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-25T03:41:21,305 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, REOPEN/MOVE 2024-11-25T03:41:21,306 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true 2024-11-25T03:41:21,306 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 2024-11-25T03:41:21,306 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 2024-11-25T03:41:21,306 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T03:41:21,306 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d73269e94d72,33145,1732506072477 2024-11-25T03:41:21,306 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 2024-11-25T03:41:21,306 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 2024-11-25T03:41:21,306 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN 2024-11-25T03:41:21,306 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN 2024-11-25T03:41:21,306 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN 2024-11-25T03:41:21,306 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4705fa4ae38907dd8b9783be2dff29da, ASSIGN 2024-11-25T03:41:21,306 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,306 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,306 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,306 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,306 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,307 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,307 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,307 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,307 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN 2024-11-25T03:41:21,307 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,307 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN 2024-11-25T03:41:21,307 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,307 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN 2024-11-25T03:41:21,307 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4705fa4ae38907dd8b9783be2dff29da, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,307 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4705fa4ae38907dd8b9783be2dff29da, ASSIGN 2024-11-25T03:41:21,307 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-25T03:41:21,308 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=65 2024-11-25T03:41:21,308 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=d73269e94d72,33145,1732506072477, table=hbase:meta, region=1588230740 2024-11-25T03:41:21,308 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d73269e94d72,33145,1732506072477, state=OPEN 2024-11-25T03:41:21,324 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33145-0x1017014aa590002, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:21,324 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40325-0x1017014aa590000, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:21,324 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38397-0x1017014aa590003, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:21,324 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:21,324 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:21,324 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:21,325 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN to state=OFFLINE, location=null, table=test, region=857a6b6999e909a9b73e698c69291205 to restore RIT 2024-11-25T03:41:21,325 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN to state=OFFLINE, location=null, table=test, region=3b442507eaac3b5222a3c4f97ff80208 to restore RIT 2024-11-25T03:41:21,325 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN to state=OFFLINE, location=null, table=test, region=922ebf14d68aba40da3a2e99777c777b to restore RIT 2024-11-25T03:41:21,325 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4705fa4ae38907dd8b9783be2dff29da, ASSIGN to state=OFFLINE, location=null, table=test, region=4705fa4ae38907dd8b9783be2dff29da to restore RIT 2024-11-25T03:41:21,326 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732506111326 2024-11-25T03:41:21,326 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-25T03:41:21,326 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T03:41:21,327 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-25T03:41:21,331 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-25T03:41:21,331 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN 2024-11-25T03:41:21,331 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T03:41:21,332 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T03:41:21,332 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3b442507eaac3b5222a3c4f97ff80208, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,332 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c73ee0e3b504288792edda3ec9a26049, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,332 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e3a490d70b339ac73875ea7638f934f, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,333 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85a5fab26752f98e6966cd407e31db69, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,333 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a7d4164efdaed87f07b6f76a94819b3, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,333 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bba1924bf10ad03aeb55628aeffc2363, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,333 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=857a6b6999e909a9b73e698c69291205, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,333 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e791add66a4fee50be34aa83a6d502a, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,333 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c0b379dd00b53bde73bca0e12f321ce, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,333 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5f64cb2042c701322ac00e312f65f16, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,333 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4b8682bf62ac734aa4174eba4ee1ddc, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,333 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb9ff3be6d639a626d88387efa9120fd, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,334 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0c260efb5b2a8a6137010d439efc8d47, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,334 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4705fa4ae38907dd8b9783be2dff29da, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,334 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5da71c9c32aa33cf6f067ea6d82ae84f, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,334 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fc93e882c2b7c5acdb53e9cf1071dc8, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,334 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=922ebf14d68aba40da3a2e99777c777b, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,334 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=745356a682ec14d9f6e2066d456cab06, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,334 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0698446036b7ae35179ebc9b07274db5, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,334 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b5ae4b02f22b24f40505bb660957004, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,334 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6aed1980a8b63b44c776d0990f923aa7, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,334 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=706facb174cc9547b71f08aa672ddcee, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,335 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b224c7c31e6ff664ce67ac884f573d7, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,335 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f812dacea03120e7730979dc37e38c6, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,335 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df71ad9490ccea77dfed73e590672db8, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,335 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c769ca025a015554efcc98619008be96, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,335 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-25T03:41:21,335 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732506141335 2024-11-25T03:41:21,335 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732506201335 2024-11-25T03:41:21,335 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-25T03:41:21,335 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-25T03:41:21,335 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T03:41:21,335 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-25T03:41:21,335 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-25T03:41:21,336 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-25T03:41:21,336 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-25T03:41:21,336 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-25T03:41:21,336 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-25T03:41:21,337 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-25T03:41:21,338 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-25T03:41:21,338 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-25T03:41:21,338 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-25T03:41:21,339 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,339 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,339 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,339 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-25T03:41:21,339 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-25T03:41:21,339 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,339 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-25T03:41:21,339 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-25T03:41:21,339 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-25T03:41:21,340 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, REOPEN/MOVE 2024-11-25T03:41:21,340 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true 2024-11-25T03:41:21,341 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 2024-11-25T03:41:21,341 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 2024-11-25T03:41:21,341 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T03:41:21,341 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d73269e94d72,33145,1732506072477 2024-11-25T03:41:21,341 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 2024-11-25T03:41:21,341 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 2024-11-25T03:41:21,341 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN 2024-11-25T03:41:21,341 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN 2024-11-25T03:41:21,341 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN 2024-11-25T03:41:21,341 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,341 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,341 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,341 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,341 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,341 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,341 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,341 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,341 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN 2024-11-25T03:41:21,341 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,341 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN 2024-11-25T03:41:21,341 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,342 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN 2024-11-25T03:41:21,342 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-25T03:41:21,342 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40325 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T03:41:21,343 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=65 2024-11-25T03:41:21,343 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=d73269e94d72,33145,1732506072477, table=hbase:meta, region=1588230740 2024-11-25T03:41:21,343 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40325 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T03:41:21,343 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d73269e94d72,33145,1732506072477, state=OPEN 2024-11-25T03:41:21,355 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38397-0x1017014aa590003, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:21,355 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40325-0x1017014aa590000, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:21,355 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33145-0x1017014aa590002, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:21,356 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:21,356 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:21,356 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:21,356 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN to state=OFFLINE, location=null, table=test, region=857a6b6999e909a9b73e698c69291205 to restore RIT 2024-11-25T03:41:21,357 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN to state=OFFLINE, location=null, table=test, region=3b442507eaac3b5222a3c4f97ff80208 to restore RIT 2024-11-25T03:41:21,357 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=922ebf14d68aba40da3a2e99777c777b, ASSIGN to state=OFFLINE, location=null, table=test, region=922ebf14d68aba40da3a2e99777c777b to restore RIT 2024-11-25T03:41:21,357 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732506111357 2024-11-25T03:41:21,357 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-25T03:41:21,357 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T03:41:21,358 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-25T03:41:21,359 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-25T03:41:21,359 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN 2024-11-25T03:41:21,359 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T03:41:21,359 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T03:41:21,363 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3b442507eaac3b5222a3c4f97ff80208, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,363 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c73ee0e3b504288792edda3ec9a26049, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,364 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e3a490d70b339ac73875ea7638f934f, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,364 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85a5fab26752f98e6966cd407e31db69, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,364 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a7d4164efdaed87f07b6f76a94819b3, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,364 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bba1924bf10ad03aeb55628aeffc2363, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,364 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=857a6b6999e909a9b73e698c69291205, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,365 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e791add66a4fee50be34aa83a6d502a, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,365 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c0b379dd00b53bde73bca0e12f321ce, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,365 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5f64cb2042c701322ac00e312f65f16, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,365 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4b8682bf62ac734aa4174eba4ee1ddc, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,365 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb9ff3be6d639a626d88387efa9120fd, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,365 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0c260efb5b2a8a6137010d439efc8d47, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,365 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4705fa4ae38907dd8b9783be2dff29da, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,365 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5da71c9c32aa33cf6f067ea6d82ae84f, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,366 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fc93e882c2b7c5acdb53e9cf1071dc8, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,366 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=922ebf14d68aba40da3a2e99777c777b, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,366 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=745356a682ec14d9f6e2066d456cab06, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,366 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0698446036b7ae35179ebc9b07274db5, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,366 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b5ae4b02f22b24f40505bb660957004, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,366 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6aed1980a8b63b44c776d0990f923aa7, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,366 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=706facb174cc9547b71f08aa672ddcee, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,366 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b224c7c31e6ff664ce67ac884f573d7, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,367 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f812dacea03120e7730979dc37e38c6, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,367 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df71ad9490ccea77dfed73e590672db8, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,367 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c769ca025a015554efcc98619008be96, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,367 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-25T03:41:21,367 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732506141367 2024-11-25T03:41:21,367 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732506201367 2024-11-25T03:41:21,367 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-25T03:41:21,367 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-25T03:41:21,367 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T03:41:21,367 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-25T03:41:21,368 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-25T03:41:21,368 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-25T03:41:21,368 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-25T03:41:21,368 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-25T03:41:21,368 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-25T03:41:21,369 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-25T03:41:21,371 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-25T03:41:21,371 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-25T03:41:21,371 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-25T03:41:21,372 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,372 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,372 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-25T03:41:21,372 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-25T03:41:21,372 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,372 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-25T03:41:21,372 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-25T03:41:21,373 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-25T03:41:21,373 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, REOPEN/MOVE 2024-11-25T03:41:21,374 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true 2024-11-25T03:41:21,375 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 2024-11-25T03:41:21,375 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 2024-11-25T03:41:21,375 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T03:41:21,375 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d73269e94d72,33145,1732506072477 2024-11-25T03:41:21,375 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 2024-11-25T03:41:21,375 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 2024-11-25T03:41:21,375 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN 2024-11-25T03:41:21,375 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN 2024-11-25T03:41:21,375 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,375 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,375 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,375 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,375 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,375 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,376 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,376 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,376 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN 2024-11-25T03:41:21,376 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,376 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN 2024-11-25T03:41:21,376 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-25T03:41:21,377 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=65 2024-11-25T03:41:21,377 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=d73269e94d72,33145,1732506072477, table=hbase:meta, region=1588230740 2024-11-25T03:41:21,377 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d73269e94d72,33145,1732506072477, state=OPEN 2024-11-25T03:41:21,387 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38397-0x1017014aa590003, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:21,387 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33145-0x1017014aa590002, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:21,387 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40325-0x1017014aa590000, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:21,387 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:21,387 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:21,387 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:21,388 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN to state=OFFLINE, location=null, table=test, region=857a6b6999e909a9b73e698c69291205 to restore RIT 2024-11-25T03:41:21,388 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3b442507eaac3b5222a3c4f97ff80208, ASSIGN to state=OFFLINE, location=null, table=test, region=3b442507eaac3b5222a3c4f97ff80208 to restore RIT 2024-11-25T03:41:21,388 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732506111388 2024-11-25T03:41:21,388 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-25T03:41:21,388 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T03:41:21,391 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-25T03:41:21,392 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-25T03:41:21,392 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN 2024-11-25T03:41:21,392 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T03:41:21,393 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T03:41:21,395 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3b442507eaac3b5222a3c4f97ff80208, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,395 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c73ee0e3b504288792edda3ec9a26049, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,395 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e3a490d70b339ac73875ea7638f934f, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,395 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85a5fab26752f98e6966cd407e31db69, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,396 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a7d4164efdaed87f07b6f76a94819b3, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,396 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bba1924bf10ad03aeb55628aeffc2363, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,396 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=857a6b6999e909a9b73e698c69291205, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,396 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e791add66a4fee50be34aa83a6d502a, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,396 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c0b379dd00b53bde73bca0e12f321ce, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,396 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5f64cb2042c701322ac00e312f65f16, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,396 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4b8682bf62ac734aa4174eba4ee1ddc, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,396 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb9ff3be6d639a626d88387efa9120fd, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,397 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0c260efb5b2a8a6137010d439efc8d47, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,397 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4705fa4ae38907dd8b9783be2dff29da, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,397 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5da71c9c32aa33cf6f067ea6d82ae84f, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,397 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fc93e882c2b7c5acdb53e9cf1071dc8, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,397 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=922ebf14d68aba40da3a2e99777c777b, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,397 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=745356a682ec14d9f6e2066d456cab06, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,397 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0698446036b7ae35179ebc9b07274db5, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,398 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b5ae4b02f22b24f40505bb660957004, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,398 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6aed1980a8b63b44c776d0990f923aa7, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,398 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=706facb174cc9547b71f08aa672ddcee, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,398 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b224c7c31e6ff664ce67ac884f573d7, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,398 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f812dacea03120e7730979dc37e38c6, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,398 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df71ad9490ccea77dfed73e590672db8, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,398 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c769ca025a015554efcc98619008be96, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,398 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-25T03:41:21,398 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732506141398 2024-11-25T03:41:21,399 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732506201398 2024-11-25T03:41:21,399 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-11-25T03:41:21,399 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-25T03:41:21,399 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T03:41:21,399 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-25T03:41:21,399 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-25T03:41:21,399 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-25T03:41:21,399 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-25T03:41:21,399 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-25T03:41:21,400 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-25T03:41:21,401 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-25T03:41:21,403 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-25T03:41:21,403 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-25T03:41:21,403 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-25T03:41:21,403 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,403 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-25T03:41:21,403 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-25T03:41:21,403 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,403 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-25T03:41:21,404 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-25T03:41:21,404 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-25T03:41:21,405 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, REOPEN/MOVE 2024-11-25T03:41:21,406 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true 2024-11-25T03:41:21,406 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 2024-11-25T03:41:21,406 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 2024-11-25T03:41:21,406 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T03:41:21,406 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d73269e94d72,33145,1732506072477 2024-11-25T03:41:21,406 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 2024-11-25T03:41:21,406 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 2024-11-25T03:41:21,406 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN 2024-11-25T03:41:21,406 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,406 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,407 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,407 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,407 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,407 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,407 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,407 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,407 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN 2024-11-25T03:41:21,407 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-25T03:41:21,408 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=65 2024-11-25T03:41:21,408 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=d73269e94d72,33145,1732506072477, table=hbase:meta, region=1588230740 2024-11-25T03:41:21,408 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d73269e94d72,33145,1732506072477, state=OPEN 2024-11-25T03:41:21,419 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33145-0x1017014aa590002, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:21,419 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38397-0x1017014aa590003, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:21,419 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40325-0x1017014aa590000, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:21,419 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:21,419 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:21,419 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:21,419 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=857a6b6999e909a9b73e698c69291205, ASSIGN to state=OFFLINE, location=null, table=test, region=857a6b6999e909a9b73e698c69291205 to restore RIT 2024-11-25T03:41:21,420 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732506111420 2024-11-25T03:41:21,420 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-25T03:41:21,420 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T03:41:21,422 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-25T03:41:21,423 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-25T03:41:21,423 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 2024-11-25T03:41:21,423 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T03:41:21,423 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T03:41:21,426 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3b442507eaac3b5222a3c4f97ff80208, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,426 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c73ee0e3b504288792edda3ec9a26049, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,427 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e3a490d70b339ac73875ea7638f934f, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,427 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85a5fab26752f98e6966cd407e31db69, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,427 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a7d4164efdaed87f07b6f76a94819b3, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,427 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bba1924bf10ad03aeb55628aeffc2363, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,427 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=857a6b6999e909a9b73e698c69291205, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,428 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e791add66a4fee50be34aa83a6d502a, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,428 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c0b379dd00b53bde73bca0e12f321ce, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,428 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5f64cb2042c701322ac00e312f65f16, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,428 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4b8682bf62ac734aa4174eba4ee1ddc, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,428 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb9ff3be6d639a626d88387efa9120fd, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,428 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0c260efb5b2a8a6137010d439efc8d47, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,428 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4705fa4ae38907dd8b9783be2dff29da, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,428 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5da71c9c32aa33cf6f067ea6d82ae84f, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,429 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fc93e882c2b7c5acdb53e9cf1071dc8, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,429 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=922ebf14d68aba40da3a2e99777c777b, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,429 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=745356a682ec14d9f6e2066d456cab06, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,429 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0698446036b7ae35179ebc9b07274db5, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,429 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b5ae4b02f22b24f40505bb660957004, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,429 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6aed1980a8b63b44c776d0990f923aa7, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,429 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=706facb174cc9547b71f08aa672ddcee, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,429 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b224c7c31e6ff664ce67ac884f573d7, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,430 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f812dacea03120e7730979dc37e38c6, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,430 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df71ad9490ccea77dfed73e590672db8, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,430 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c769ca025a015554efcc98619008be96, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,430 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-25T03:41:21,430 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732506141430 2024-11-25T03:41:21,430 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732506201430 2024-11-25T03:41:21,430 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-11-25T03:41:21,430 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-25T03:41:21,430 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T03:41:21,430 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-25T03:41:21,430 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-25T03:41:21,431 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-25T03:41:21,431 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-25T03:41:21,431 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-25T03:41:21,431 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-25T03:41:21,432 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-25T03:41:21,434 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-25T03:41:21,434 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-25T03:41:21,434 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-25T03:41:21,434 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-25T03:41:21,434 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-25T03:41:21,434 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,434 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-25T03:41:21,434 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-25T03:41:21,435 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-25T03:41:21,435 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, REOPEN/MOVE 2024-11-25T03:41:21,436 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true 2024-11-25T03:41:21,436 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 2024-11-25T03:41:21,436 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 2024-11-25T03:41:21,436 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T03:41:21,436 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d73269e94d72,33145,1732506072477 2024-11-25T03:41:21,436 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 2024-11-25T03:41:21,436 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 2024-11-25T03:41:21,436 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,436 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,437 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,437 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,437 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,437 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,437 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,437 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-25T03:41:21,438 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=65 2024-11-25T03:41:21,438 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=d73269e94d72,33145,1732506072477, table=hbase:meta, region=1588230740 2024-11-25T03:41:21,438 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d73269e94d72,33145,1732506072477, state=OPEN 2024-11-25T03:41:21,446 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40325 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T03:41:21,450 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38397-0x1017014aa590003, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:21,450 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33145-0x1017014aa590002, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:21,450 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40325-0x1017014aa590000, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:21,450 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:21,450 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:21,450 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:21,451 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732506111451 2024-11-25T03:41:21,451 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-25T03:41:21,451 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T03:41:21,453 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-25T03:41:21,454 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-25T03:41:21,454 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 2024-11-25T03:41:21,454 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T03:41:21,455 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T03:41:21,458 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3b442507eaac3b5222a3c4f97ff80208, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,458 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c73ee0e3b504288792edda3ec9a26049, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,458 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e3a490d70b339ac73875ea7638f934f, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,458 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85a5fab26752f98e6966cd407e31db69, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,458 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a7d4164efdaed87f07b6f76a94819b3, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,458 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bba1924bf10ad03aeb55628aeffc2363, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,459 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=857a6b6999e909a9b73e698c69291205, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,459 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e791add66a4fee50be34aa83a6d502a, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,459 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c0b379dd00b53bde73bca0e12f321ce, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,459 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5f64cb2042c701322ac00e312f65f16, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,459 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4b8682bf62ac734aa4174eba4ee1ddc, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,459 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb9ff3be6d639a626d88387efa9120fd, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,459 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0c260efb5b2a8a6137010d439efc8d47, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,459 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4705fa4ae38907dd8b9783be2dff29da, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,459 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5da71c9c32aa33cf6f067ea6d82ae84f, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,460 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fc93e882c2b7c5acdb53e9cf1071dc8, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,460 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=922ebf14d68aba40da3a2e99777c777b, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,460 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=745356a682ec14d9f6e2066d456cab06, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,460 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0698446036b7ae35179ebc9b07274db5, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,460 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b5ae4b02f22b24f40505bb660957004, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,460 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6aed1980a8b63b44c776d0990f923aa7, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,460 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=706facb174cc9547b71f08aa672ddcee, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,460 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b224c7c31e6ff664ce67ac884f573d7, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,460 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f812dacea03120e7730979dc37e38c6, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,461 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df71ad9490ccea77dfed73e590672db8, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,461 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c769ca025a015554efcc98619008be96, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,461 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-25T03:41:21,461 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732506141461 2024-11-25T03:41:21,461 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732506201461 2024-11-25T03:41:21,461 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-25T03:41:21,461 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-25T03:41:21,461 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T03:41:21,461 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-25T03:41:21,461 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-25T03:41:21,462 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-25T03:41:21,462 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-25T03:41:21,462 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-25T03:41:21,462 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-25T03:41:21,463 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-25T03:41:21,465 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-25T03:41:21,465 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-25T03:41:21,465 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-25T03:41:21,465 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-25T03:41:21,465 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,465 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-25T03:41:21,465 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-25T03:41:21,466 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-25T03:41:21,466 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, REOPEN/MOVE 2024-11-25T03:41:21,467 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true 2024-11-25T03:41:21,467 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 2024-11-25T03:41:21,467 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 2024-11-25T03:41:21,467 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T03:41:21,467 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d73269e94d72,33145,1732506072477 2024-11-25T03:41:21,467 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 2024-11-25T03:41:21,467 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,467 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,467 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,467 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,467 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,467 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.1732506074123, worker=d73269e94d72,38397,1732506072528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,468 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-25T03:41:21,468 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=65 2024-11-25T03:41:21,468 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=d73269e94d72,33145,1732506072477, table=hbase:meta, region=1588230740 2024-11-25T03:41:21,469 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d73269e94d72,33145,1732506072477, state=OPEN 2024-11-25T03:41:21,482 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38397-0x1017014aa590003, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:21,482 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40325-0x1017014aa590000, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:21,482 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:21,482 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:21,482 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33145-0x1017014aa590002, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:21,482 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:21,491 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732506111490 2024-11-25T03:41:21,491 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-25T03:41:21,491 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T03:41:21,492 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-25T03:41:21,493 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-25T03:41:21,493 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d73269e94d72,33145,1732506072477 2024-11-25T03:41:21,493 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T03:41:21,493 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d73269e94d72,33145,1732506072477 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T03:41:21,497 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3b442507eaac3b5222a3c4f97ff80208, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,497 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c73ee0e3b504288792edda3ec9a26049, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,497 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e3a490d70b339ac73875ea7638f934f, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,497 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85a5fab26752f98e6966cd407e31db69, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,497 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a7d4164efdaed87f07b6f76a94819b3, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,497 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bba1924bf10ad03aeb55628aeffc2363, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,497 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=857a6b6999e909a9b73e698c69291205, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,498 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e791add66a4fee50be34aa83a6d502a, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,498 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c0b379dd00b53bde73bca0e12f321ce, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,498 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5f64cb2042c701322ac00e312f65f16, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,498 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4b8682bf62ac734aa4174eba4ee1ddc, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,498 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb9ff3be6d639a626d88387efa9120fd, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,498 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0c260efb5b2a8a6137010d439efc8d47, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,498 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4705fa4ae38907dd8b9783be2dff29da, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,498 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5da71c9c32aa33cf6f067ea6d82ae84f, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,498 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fc93e882c2b7c5acdb53e9cf1071dc8, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,498 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=922ebf14d68aba40da3a2e99777c777b, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,498 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=745356a682ec14d9f6e2066d456cab06, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,498 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0698446036b7ae35179ebc9b07274db5, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,498 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b5ae4b02f22b24f40505bb660957004, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,499 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6aed1980a8b63b44c776d0990f923aa7, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,499 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=706facb174cc9547b71f08aa672ddcee, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,499 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b224c7c31e6ff664ce67ac884f573d7, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,499 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f812dacea03120e7730979dc37e38c6, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,499 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df71ad9490ccea77dfed73e590672db8, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,499 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c769ca025a015554efcc98619008be96, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,499 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-25T03:41:21,499 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732506141499 2024-11-25T03:41:21,499 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732506201499 2024-11-25T03:41:21,499 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-25T03:41:21,499 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-25T03:41:21,499 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T03:41:21,499 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-25T03:41:21,499 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-25T03:41:21,500 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-25T03:41:21,500 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-25T03:41:21,500 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-25T03:41:21,500 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-25T03:41:21,501 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-25T03:41:21,502 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-25T03:41:21,502 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-25T03:41:21,502 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-25T03:41:21,502 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,502 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-25T03:41:21,502 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-25T03:41:21,503 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-25T03:41:21,503 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, REOPEN/MOVE 2024-11-25T03:41:21,503 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true 2024-11-25T03:41:21,503 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 2024-11-25T03:41:21,504 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 2024-11-25T03:41:21,504 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T03:41:21,504 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d73269e94d72,33145,1732506072477 2024-11-25T03:41:21,504 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,504 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,504 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,504 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,504 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,504 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-25T03:41:21,505 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=65 2024-11-25T03:41:21,505 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=d73269e94d72,33145,1732506072477, table=hbase:meta, region=1588230740 2024-11-25T03:41:21,505 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d73269e94d72,33145,1732506072477, state=OPEN 2024-11-25T03:41:21,513 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38397-0x1017014aa590003, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:21,513 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33145-0x1017014aa590002, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:21,513 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40325-0x1017014aa590000, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:21,513 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:21,513 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:21,513 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:21,514 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732506111514 2024-11-25T03:41:21,514 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-25T03:41:21,514 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T03:41:21,516 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-25T03:41:21,517 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-25T03:41:21,517 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T03:41:21,517 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T03:41:21,517 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T03:41:21,520 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3b442507eaac3b5222a3c4f97ff80208, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,520 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c73ee0e3b504288792edda3ec9a26049, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,520 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e3a490d70b339ac73875ea7638f934f, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,520 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85a5fab26752f98e6966cd407e31db69, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,520 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a7d4164efdaed87f07b6f76a94819b3, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,520 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bba1924bf10ad03aeb55628aeffc2363, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,520 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=857a6b6999e909a9b73e698c69291205, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,520 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e791add66a4fee50be34aa83a6d502a, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,520 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c0b379dd00b53bde73bca0e12f321ce, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,521 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5f64cb2042c701322ac00e312f65f16, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,521 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4b8682bf62ac734aa4174eba4ee1ddc, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,521 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb9ff3be6d639a626d88387efa9120fd, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,521 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0c260efb5b2a8a6137010d439efc8d47, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,521 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4705fa4ae38907dd8b9783be2dff29da, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,521 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5da71c9c32aa33cf6f067ea6d82ae84f, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,521 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fc93e882c2b7c5acdb53e9cf1071dc8, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,521 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=922ebf14d68aba40da3a2e99777c777b, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,521 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=745356a682ec14d9f6e2066d456cab06, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,521 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0698446036b7ae35179ebc9b07274db5, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,521 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b5ae4b02f22b24f40505bb660957004, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,521 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6aed1980a8b63b44c776d0990f923aa7, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,521 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=706facb174cc9547b71f08aa672ddcee, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,522 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b224c7c31e6ff664ce67ac884f573d7, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,522 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f812dacea03120e7730979dc37e38c6, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,522 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df71ad9490ccea77dfed73e590672db8, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,522 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c769ca025a015554efcc98619008be96, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,522 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-25T03:41:21,522 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732506141522 2024-11-25T03:41:21,522 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732506201522 2024-11-25T03:41:21,522 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-25T03:41:21,522 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-25T03:41:21,522 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T03:41:21,522 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-25T03:41:21,522 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-25T03:41:21,522 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-25T03:41:21,522 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-25T03:41:21,522 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-25T03:41:21,523 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-25T03:41:21,523 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-25T03:41:21,525 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-25T03:41:21,525 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-25T03:41:21,525 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-25T03:41:21,525 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-25T03:41:21,525 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-25T03:41:21,525 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-25T03:41:21,526 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, REOPEN/MOVE 2024-11-25T03:41:21,526 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true 2024-11-25T03:41:21,526 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 2024-11-25T03:41:21,526 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 2024-11-25T03:41:21,526 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T03:41:21,526 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,526 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,526 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,526 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,526 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-25T03:41:21,527 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=65 2024-11-25T03:41:21,527 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=d73269e94d72,33145,1732506072477, table=hbase:meta, region=1588230740 2024-11-25T03:41:21,527 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d73269e94d72,33145,1732506072477, state=OPEN 2024-11-25T03:41:21,534 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38397-0x1017014aa590003, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:21,534 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40325-0x1017014aa590000, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:21,534 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33145-0x1017014aa590002, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:21,534 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:21,534 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:21,534 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:21,535 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732506111535 2024-11-25T03:41:21,535 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-25T03:41:21,535 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T03:41:21,537 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-25T03:41:21,537 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-25T03:41:21,538 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 2024-11-25T03:41:21,538 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T03:41:21,538 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T03:41:21,540 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3b442507eaac3b5222a3c4f97ff80208, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,540 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c73ee0e3b504288792edda3ec9a26049, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,540 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e3a490d70b339ac73875ea7638f934f, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,540 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85a5fab26752f98e6966cd407e31db69, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,540 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a7d4164efdaed87f07b6f76a94819b3, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,540 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bba1924bf10ad03aeb55628aeffc2363, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,540 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=857a6b6999e909a9b73e698c69291205, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,540 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e791add66a4fee50be34aa83a6d502a, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,541 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c0b379dd00b53bde73bca0e12f321ce, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,541 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5f64cb2042c701322ac00e312f65f16, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,541 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4b8682bf62ac734aa4174eba4ee1ddc, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,541 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb9ff3be6d639a626d88387efa9120fd, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,541 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0c260efb5b2a8a6137010d439efc8d47, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,541 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4705fa4ae38907dd8b9783be2dff29da, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,541 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5da71c9c32aa33cf6f067ea6d82ae84f, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,541 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fc93e882c2b7c5acdb53e9cf1071dc8, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,541 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=922ebf14d68aba40da3a2e99777c777b, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,541 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=745356a682ec14d9f6e2066d456cab06, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,541 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0698446036b7ae35179ebc9b07274db5, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,541 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b5ae4b02f22b24f40505bb660957004, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,541 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6aed1980a8b63b44c776d0990f923aa7, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,542 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=706facb174cc9547b71f08aa672ddcee, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,542 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b224c7c31e6ff664ce67ac884f573d7, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,542 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f812dacea03120e7730979dc37e38c6, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,542 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df71ad9490ccea77dfed73e590672db8, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,542 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c769ca025a015554efcc98619008be96, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,542 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-25T03:41:21,542 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732506141542 2024-11-25T03:41:21,542 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732506201542 2024-11-25T03:41:21,542 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-11-25T03:41:21,542 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-25T03:41:21,542 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T03:41:21,542 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-25T03:41:21,542 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-25T03:41:21,542 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-25T03:41:21,543 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-25T03:41:21,543 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-25T03:41:21,543 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-25T03:41:21,543 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-25T03:41:21,545 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-25T03:41:21,545 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-25T03:41:21,545 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-25T03:41:21,545 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-25T03:41:21,545 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-25T03:41:21,545 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, REOPEN/MOVE 2024-11-25T03:41:21,546 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true 2024-11-25T03:41:21,546 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 2024-11-25T03:41:21,546 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 2024-11-25T03:41:21,546 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,546 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,546 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,546 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-25T03:41:21,547 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=65 2024-11-25T03:41:21,547 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=d73269e94d72,33145,1732506072477, table=hbase:meta, region=1588230740 2024-11-25T03:41:21,547 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d73269e94d72,33145,1732506072477, state=OPEN 2024-11-25T03:41:21,552 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40325 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T03:41:21,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33145-0x1017014aa590002, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:21,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40325-0x1017014aa590000, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:21,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38397-0x1017014aa590003, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:21,555 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:21,555 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:21,555 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:21,557 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732506111556 2024-11-25T03:41:21,557 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-25T03:41:21,557 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T03:41:21,558 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-25T03:41:21,559 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-25T03:41:21,559 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 2024-11-25T03:41:21,559 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T03:41:21,559 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T03:41:21,561 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3b442507eaac3b5222a3c4f97ff80208, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,561 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c73ee0e3b504288792edda3ec9a26049, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,561 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e3a490d70b339ac73875ea7638f934f, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,561 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85a5fab26752f98e6966cd407e31db69, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,562 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a7d4164efdaed87f07b6f76a94819b3, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,562 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bba1924bf10ad03aeb55628aeffc2363, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,562 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=857a6b6999e909a9b73e698c69291205, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,562 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e791add66a4fee50be34aa83a6d502a, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,562 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c0b379dd00b53bde73bca0e12f321ce, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,562 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5f64cb2042c701322ac00e312f65f16, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,562 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4b8682bf62ac734aa4174eba4ee1ddc, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,562 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb9ff3be6d639a626d88387efa9120fd, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,562 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0c260efb5b2a8a6137010d439efc8d47, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,562 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4705fa4ae38907dd8b9783be2dff29da, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,562 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5da71c9c32aa33cf6f067ea6d82ae84f, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,562 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fc93e882c2b7c5acdb53e9cf1071dc8, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,562 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=922ebf14d68aba40da3a2e99777c777b, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,563 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=745356a682ec14d9f6e2066d456cab06, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,563 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0698446036b7ae35179ebc9b07274db5, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,563 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b5ae4b02f22b24f40505bb660957004, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,563 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6aed1980a8b63b44c776d0990f923aa7, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,563 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=706facb174cc9547b71f08aa672ddcee, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,563 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b224c7c31e6ff664ce67ac884f573d7, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,563 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f812dacea03120e7730979dc37e38c6, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,563 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df71ad9490ccea77dfed73e590672db8, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,563 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c769ca025a015554efcc98619008be96, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,563 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-25T03:41:21,563 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732506141563 2024-11-25T03:41:21,563 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732506201563 2024-11-25T03:41:21,563 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-11-25T03:41:21,563 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-25T03:41:21,563 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T03:41:21,563 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-25T03:41:21,564 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-25T03:41:21,564 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-25T03:41:21,564 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-25T03:41:21,564 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-25T03:41:21,564 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-25T03:41:21,565 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-25T03:41:21,566 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-25T03:41:21,566 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-25T03:41:21,566 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-25T03:41:21,567 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-25T03:41:21,567 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, REOPEN/MOVE 2024-11-25T03:41:21,567 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true 2024-11-25T03:41:21,567 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 2024-11-25T03:41:21,567 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,567 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C35981%2C1732506072374.meta.1732506079074.meta, worker=d73269e94d72,33145,1732506072477 didn't hold the lock before restarting, skip acquiring lock. 2024-11-25T03:41:21,568 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-25T03:41:21,568 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=65 2024-11-25T03:41:21,568 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=d73269e94d72,33145,1732506072477, table=hbase:meta, region=1588230740 2024-11-25T03:41:21,568 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d73269e94d72,33145,1732506072477, state=OPEN 2024-11-25T03:41:21,576 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38397-0x1017014aa590003, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:21,576 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40325-0x1017014aa590000, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:21,576 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33145-0x1017014aa590002, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:21,576 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:21,577 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:21,576 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:21,578 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732506111578 2024-11-25T03:41:21,578 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-25T03:41:21,578 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T03:41:21,579 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-25T03:41:21,580 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-25T03:41:21,580 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true 2024-11-25T03:41:21,580 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T03:41:21,581 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1604) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T03:41:21,582 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3b442507eaac3b5222a3c4f97ff80208, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,583 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c73ee0e3b504288792edda3ec9a26049, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,583 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e3a490d70b339ac73875ea7638f934f, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,583 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85a5fab26752f98e6966cd407e31db69, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,583 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a7d4164efdaed87f07b6f76a94819b3, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,583 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bba1924bf10ad03aeb55628aeffc2363, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,583 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=857a6b6999e909a9b73e698c69291205, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,583 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e791add66a4fee50be34aa83a6d502a, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,583 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c0b379dd00b53bde73bca0e12f321ce, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,583 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5f64cb2042c701322ac00e312f65f16, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,583 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4b8682bf62ac734aa4174eba4ee1ddc, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,583 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb9ff3be6d639a626d88387efa9120fd, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,583 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0c260efb5b2a8a6137010d439efc8d47, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,584 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4705fa4ae38907dd8b9783be2dff29da, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,584 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5da71c9c32aa33cf6f067ea6d82ae84f, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,584 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fc93e882c2b7c5acdb53e9cf1071dc8, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,584 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=922ebf14d68aba40da3a2e99777c777b, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,584 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=745356a682ec14d9f6e2066d456cab06, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,584 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0698446036b7ae35179ebc9b07274db5, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,584 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b5ae4b02f22b24f40505bb660957004, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,584 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6aed1980a8b63b44c776d0990f923aa7, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,584 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=706facb174cc9547b71f08aa672ddcee, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,584 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b224c7c31e6ff664ce67ac884f573d7, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,584 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f812dacea03120e7730979dc37e38c6, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,584 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df71ad9490ccea77dfed73e590672db8, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,584 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c769ca025a015554efcc98619008be96, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,585 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-25T03:41:21,585 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732506141585 2024-11-25T03:41:21,585 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732506201585 2024-11-25T03:41:21,585 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-11-25T03:41:21,585 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-25T03:41:21,585 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T03:41:21,585 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-25T03:41:21,585 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-25T03:41:21,585 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-25T03:41:21,585 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-25T03:41:21,585 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-25T03:41:21,585 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-25T03:41:21,586 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-25T03:41:21,587 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-25T03:41:21,587 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-25T03:41:21,588 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-25T03:41:21,588 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, REOPEN/MOVE 2024-11-25T03:41:21,588 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true 2024-11-25T03:41:21,588 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-25T03:41:21,589 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-25T03:41:21,589 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=65 2024-11-25T03:41:21,589 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=d73269e94d72,33145,1732506072477, table=hbase:meta, region=1588230740 2024-11-25T03:41:21,589 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d73269e94d72,33145,1732506072477, state=OPEN 2024-11-25T03:41:21,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38397-0x1017014aa590003, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:21,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40325-0x1017014aa590000, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:21,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33145-0x1017014aa590002, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:41:21,597 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:21,597 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:21,597 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:41:21,598 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732506111598 2024-11-25T03:41:21,598 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-25T03:41:21,598 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T03:41:21,600 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-25T03:41:21,601 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=60, state=ROLLEDBACK, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=76, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3b442507eaac3b5222a3c4f97ff80208, server=d73269e94d72,33145,1732506072477:java.lang.RuntimeException: inject code bug; ServerCrashProcedure d73269e94d72,35981,1732506072374, splitWal=true, meta=true exec-time=2.2550 sec 2024-11-25T03:41:21,603 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3b442507eaac3b5222a3c4f97ff80208, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,603 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c73ee0e3b504288792edda3ec9a26049, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,603 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e3a490d70b339ac73875ea7638f934f, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,603 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85a5fab26752f98e6966cd407e31db69, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,603 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a7d4164efdaed87f07b6f76a94819b3, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,603 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bba1924bf10ad03aeb55628aeffc2363, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,603 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=857a6b6999e909a9b73e698c69291205, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,603 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5e791add66a4fee50be34aa83a6d502a, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,603 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c0b379dd00b53bde73bca0e12f321ce, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,604 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5f64cb2042c701322ac00e312f65f16, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,604 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4b8682bf62ac734aa4174eba4ee1ddc, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,604 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb9ff3be6d639a626d88387efa9120fd, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,604 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0c260efb5b2a8a6137010d439efc8d47, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,604 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4705fa4ae38907dd8b9783be2dff29da, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,604 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5da71c9c32aa33cf6f067ea6d82ae84f, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,604 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2fc93e882c2b7c5acdb53e9cf1071dc8, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,604 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=922ebf14d68aba40da3a2e99777c777b, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,604 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=745356a682ec14d9f6e2066d456cab06, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,604 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0698446036b7ae35179ebc9b07274db5, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,604 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b5ae4b02f22b24f40505bb660957004, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,604 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6aed1980a8b63b44c776d0990f923aa7, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,604 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=706facb174cc9547b71f08aa672ddcee, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,605 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b224c7c31e6ff664ce67ac884f573d7, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,605 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f812dacea03120e7730979dc37e38c6, regionState=OPEN, lastHost=d73269e94d72,33145,1732506072477, regionLocation=d73269e94d72,33145,1732506072477, openSeqNum=2 2024-11-25T03:41:21,605 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df71ad9490ccea77dfed73e590672db8, regionState=OPEN, lastHost=d73269e94d72,38397,1732506072528, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,605 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c769ca025a015554efcc98619008be96, regionState=OPENING, lastHost=d73269e94d72,35981,1732506072374, regionLocation=d73269e94d72,38397,1732506072528, openSeqNum=2 2024-11-25T03:41:21,605 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-25T03:41:21,605 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732506141605 2024-11-25T03:41:21,605 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732506201605 2024-11-25T03:41:21,605 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-11-25T03:41:21,621 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.assignment.TestRollbackSCP#testFailAndRollback Thread=352 (was 394), OpenFileDescriptor=616 (was 651), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=192 (was 192), ProcessCount=11 (was 11), AvailableMemoryMB=9217 (was 9265) 2024-11-25T03:41:21,621 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-25T03:41:21,621 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T03:41:21,621 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T03:41:21,622 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:41:21,622 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:41:21,622 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:41:21,622 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-25T03:41:21,623 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1546128933, stopped=false 2024-11-25T03:41:21,623 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=d73269e94d72,40325,1732506071573 2024-11-25T03:41:21,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40325-0x1017014aa590000, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-25T03:41:21,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33145-0x1017014aa590002, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-25T03:41:21,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38397-0x1017014aa590003, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-25T03:41:21,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33145-0x1017014aa590002, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:41:21,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40325-0x1017014aa590000, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:41:21,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38397-0x1017014aa590003, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:41:21,629 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T03:41:21,629 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T03:41:21,629 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T03:41:21,630 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:41:21,630 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'd73269e94d72,33145,1732506072477' ***** 2024-11-25T03:41:21,630 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-25T03:41:21,630 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40325-0x1017014aa590000, quorum=127.0.0.1:56467, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T03:41:21,630 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38397-0x1017014aa590003, quorum=127.0.0.1:56467, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T03:41:21,630 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'd73269e94d72,38397,1732506072528' ***** 2024-11-25T03:41:21,630 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33145-0x1017014aa590002, quorum=127.0.0.1:56467, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T03:41:21,630 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-25T03:41:21,630 INFO [RS:1;d73269e94d72:33145 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-25T03:41:21,630 INFO [RS:1;d73269e94d72:33145 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-25T03:41:21,630 INFO [RS:1;d73269e94d72:33145 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-25T03:41:21,630 INFO [RS:1;d73269e94d72:33145 {}] regionserver.HRegionServer(3091): Received CLOSE for bba1924bf10ad03aeb55628aeffc2363 2024-11-25T03:41:21,631 INFO [RS:2;d73269e94d72:38397 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-25T03:41:21,631 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-25T03:41:21,631 INFO [RS:2;d73269e94d72:38397 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-25T03:41:21,631 INFO [RS:2;d73269e94d72:38397 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-25T03:41:21,631 INFO [RS:2;d73269e94d72:38397 {}] regionserver.HRegionServer(3091): Received CLOSE for df71ad9490ccea77dfed73e590672db8 2024-11-25T03:41:21,631 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-25T03:41:21,632 INFO [RS:2;d73269e94d72:38397 {}] regionserver.HRegionServer(3091): Received CLOSE for 4c0b379dd00b53bde73bca0e12f321ce 2024-11-25T03:41:21,632 INFO [RS:2;d73269e94d72:38397 {}] regionserver.HRegionServer(3091): Received CLOSE for 4705fa4ae38907dd8b9783be2dff29da 2024-11-25T03:41:21,632 INFO [RS:1;d73269e94d72:33145 {}] regionserver.HRegionServer(3091): Received CLOSE for 745356a682ec14d9f6e2066d456cab06 2024-11-25T03:41:21,632 INFO [RS:2;d73269e94d72:38397 {}] regionserver.HRegionServer(3091): Received CLOSE for 0698446036b7ae35179ebc9b07274db5 2024-11-25T03:41:21,632 INFO [RS:2;d73269e94d72:38397 {}] regionserver.HRegionServer(3091): Received CLOSE for 857a6b6999e909a9b73e698c69291205 2024-11-25T03:41:21,632 INFO [RS:1;d73269e94d72:33145 {}] regionserver.HRegionServer(3091): Received CLOSE for 85a5fab26752f98e6966cd407e31db69 2024-11-25T03:41:21,632 INFO [RS:2;d73269e94d72:38397 {}] regionserver.HRegionServer(3091): Received CLOSE for 0c260efb5b2a8a6137010d439efc8d47 2024-11-25T03:41:21,632 INFO [RS:1;d73269e94d72:33145 {}] regionserver.HRegionServer(3091): Received CLOSE for 3b442507eaac3b5222a3c4f97ff80208 2024-11-25T03:41:21,632 INFO [RS:2;d73269e94d72:38397 {}] regionserver.HRegionServer(3091): Received CLOSE for 5e3a490d70b339ac73875ea7638f934f 2024-11-25T03:41:21,632 INFO [RS:2;d73269e94d72:38397 {}] regionserver.HRegionServer(3091): Received CLOSE for 8a7d4164efdaed87f07b6f76a94819b3 2024-11-25T03:41:21,632 INFO [RS:1;d73269e94d72:33145 {}] regionserver.HRegionServer(3091): Received CLOSE for f5f64cb2042c701322ac00e312f65f16 2024-11-25T03:41:21,632 INFO [RS:2;d73269e94d72:38397 {}] regionserver.HRegionServer(3091): Received CLOSE for 706facb174cc9547b71f08aa672ddcee 2024-11-25T03:41:21,632 INFO [RS:1;d73269e94d72:33145 {}] regionserver.HRegionServer(3091): Received CLOSE for 5da71c9c32aa33cf6f067ea6d82ae84f 2024-11-25T03:41:21,632 INFO [RS:2;d73269e94d72:38397 {}] regionserver.HRegionServer(3091): Received CLOSE for 8b5ae4b02f22b24f40505bb660957004 2024-11-25T03:41:21,632 INFO [RS:2;d73269e94d72:38397 {}] regionserver.HRegionServer(3091): Received CLOSE for c769ca025a015554efcc98619008be96 2024-11-25T03:41:21,632 INFO [RS:1;d73269e94d72:33145 {}] regionserver.HRegionServer(3091): Received CLOSE for 2b224c7c31e6ff664ce67ac884f573d7 2024-11-25T03:41:21,632 INFO [RS:2;d73269e94d72:38397 {}] regionserver.HRegionServer(3091): Received CLOSE for eb9ff3be6d639a626d88387efa9120fd 2024-11-25T03:41:21,633 INFO [RS:1;d73269e94d72:33145 {}] regionserver.HRegionServer(3091): Received CLOSE for 5e791add66a4fee50be34aa83a6d502a 2024-11-25T03:41:21,633 INFO [RS:2;d73269e94d72:38397 {}] regionserver.HRegionServer(3091): Received CLOSE for 2fc93e882c2b7c5acdb53e9cf1071dc8 2024-11-25T03:41:21,633 INFO [RS:1;d73269e94d72:33145 {}] regionserver.HRegionServer(3091): Received CLOSE for 6aed1980a8b63b44c776d0990f923aa7 2024-11-25T03:41:21,633 INFO [RS:2;d73269e94d72:38397 {}] regionserver.HRegionServer(959): stopping server d73269e94d72,38397,1732506072528 2024-11-25T03:41:21,633 INFO [RS:1;d73269e94d72:33145 {}] regionserver.HRegionServer(3091): Received CLOSE for 5f812dacea03120e7730979dc37e38c6 2024-11-25T03:41:21,633 INFO [RS:2;d73269e94d72:38397 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T03:41:21,633 INFO [RS:1;d73269e94d72:33145 {}] regionserver.HRegionServer(3091): Received CLOSE for a4b8682bf62ac734aa4174eba4ee1ddc 2024-11-25T03:41:21,633 INFO [RS:2;d73269e94d72:38397 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;d73269e94d72:38397. 2024-11-25T03:41:21,633 INFO [RS:1;d73269e94d72:33145 {}] regionserver.HRegionServer(3091): Received CLOSE for 922ebf14d68aba40da3a2e99777c777b 2024-11-25T03:41:21,633 DEBUG [RS:2;d73269e94d72:38397 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T03:41:21,633 DEBUG [RS:2;d73269e94d72:38397 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:41:21,633 INFO [RS:1;d73269e94d72:33145 {}] regionserver.HRegionServer(3091): Received CLOSE for c73ee0e3b504288792edda3ec9a26049 2024-11-25T03:41:21,633 INFO [RS:1;d73269e94d72:33145 {}] regionserver.HRegionServer(959): stopping server d73269e94d72,33145,1732506072477 2024-11-25T03:41:21,633 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing bba1924bf10ad03aeb55628aeffc2363, disabling compactions & flushes 2024-11-25T03:41:21,633 INFO [RS:2;d73269e94d72:38397 {}] regionserver.HRegionServer(1321): Waiting on 13 regions to close 2024-11-25T03:41:21,633 INFO [RS:1;d73269e94d72:33145 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T03:41:21,633 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,fff,1732506074951.bba1924bf10ad03aeb55628aeffc2363. 2024-11-25T03:41:21,633 INFO [RS:1;d73269e94d72:33145 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;d73269e94d72:33145. 2024-11-25T03:41:21,633 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1732506074951.bba1924bf10ad03aeb55628aeffc2363. 2024-11-25T03:41:21,633 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,fff,1732506074951.bba1924bf10ad03aeb55628aeffc2363. after waiting 0 ms 2024-11-25T03:41:21,633 DEBUG [RS:2;d73269e94d72:38397 {}] regionserver.HRegionServer(1351): Waiting on 0698446036b7ae35179ebc9b07274db5, 0c260efb5b2a8a6137010d439efc8d47, 2fc93e882c2b7c5acdb53e9cf1071dc8, 4705fa4ae38907dd8b9783be2dff29da, 4c0b379dd00b53bde73bca0e12f321ce, 5e3a490d70b339ac73875ea7638f934f, 706facb174cc9547b71f08aa672ddcee, 857a6b6999e909a9b73e698c69291205, 8a7d4164efdaed87f07b6f76a94819b3, 8b5ae4b02f22b24f40505bb660957004, c769ca025a015554efcc98619008be96, df71ad9490ccea77dfed73e590672db8, eb9ff3be6d639a626d88387efa9120fd 2024-11-25T03:41:21,633 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,fff,1732506074951.bba1924bf10ad03aeb55628aeffc2363. 2024-11-25T03:41:21,634 DEBUG [RS:1;d73269e94d72:33145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T03:41:21,634 DEBUG [RS:1;d73269e94d72:33145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:41:21,634 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing df71ad9490ccea77dfed73e590672db8, disabling compactions & flushes 2024-11-25T03:41:21,634 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,yyy,1732506074951.df71ad9490ccea77dfed73e590672db8. 2024-11-25T03:41:21,634 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1732506074951.df71ad9490ccea77dfed73e590672db8. 2024-11-25T03:41:21,634 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1732506074951.df71ad9490ccea77dfed73e590672db8. after waiting 0 ms 2024-11-25T03:41:21,634 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1732506074951.df71ad9490ccea77dfed73e590672db8. 2024-11-25T03:41:21,634 INFO [RS:1;d73269e94d72:33145 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-25T03:41:21,634 INFO [RS:1;d73269e94d72:33145 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-25T03:41:21,634 INFO [RS:1;d73269e94d72:33145 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-25T03:41:21,634 INFO [RS:1;d73269e94d72:33145 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-25T03:41:21,634 INFO [RS:1;d73269e94d72:33145 {}] regionserver.HRegionServer(1321): Waiting on 14 regions to close 2024-11-25T03:41:21,634 DEBUG [RS:1;d73269e94d72:33145 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 2b224c7c31e6ff664ce67ac884f573d7, 3b442507eaac3b5222a3c4f97ff80208, 5da71c9c32aa33cf6f067ea6d82ae84f, 5e791add66a4fee50be34aa83a6d502a, 5f812dacea03120e7730979dc37e38c6, 6aed1980a8b63b44c776d0990f923aa7, 745356a682ec14d9f6e2066d456cab06, 85a5fab26752f98e6966cd407e31db69, 922ebf14d68aba40da3a2e99777c777b, a4b8682bf62ac734aa4174eba4ee1ddc, bba1924bf10ad03aeb55628aeffc2363, c73ee0e3b504288792edda3ec9a26049, f5f64cb2042c701322ac00e312f65f16 2024-11-25T03:41:21,634 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-25T03:41:21,635 INFO [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-25T03:41:21,635 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-25T03:41:21,635 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-25T03:41:21,635 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-25T03:41:21,635 INFO [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.64 KB heapSize=5.42 KB 2024-11-25T03:41:21,639 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/df71ad9490ccea77dfed73e590672db8/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-25T03:41:21,640 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,yyy,1732506074951.df71ad9490ccea77dfed73e590672db8. 2024-11-25T03:41:21,640 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for df71ad9490ccea77dfed73e590672db8: Waiting for close lock at 1732506081634Running coprocessor pre-close hooks at 1732506081634Disabling compacts and flushes for region at 1732506081634Disabling writes for close at 1732506081634Writing region close event to WAL at 1732506081635 (+1 ms)Running coprocessor post-close hooks at 1732506081640 (+5 ms)Closed at 1732506081640 2024-11-25T03:41:21,640 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,yyy,1732506074951.df71ad9490ccea77dfed73e590672db8. 2024-11-25T03:41:21,640 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4c0b379dd00b53bde73bca0e12f321ce, disabling compactions & flushes 2024-11-25T03:41:21,640 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,iii,1732506074951.4c0b379dd00b53bde73bca0e12f321ce. 2024-11-25T03:41:21,640 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1732506074951.4c0b379dd00b53bde73bca0e12f321ce. 2024-11-25T03:41:21,640 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,iii,1732506074951.4c0b379dd00b53bde73bca0e12f321ce. after waiting 0 ms 2024-11-25T03:41:21,640 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,iii,1732506074951.4c0b379dd00b53bde73bca0e12f321ce. 2024-11-25T03:41:21,641 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/bba1924bf10ad03aeb55628aeffc2363/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-25T03:41:21,642 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,fff,1732506074951.bba1924bf10ad03aeb55628aeffc2363. 2024-11-25T03:41:21,642 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for bba1924bf10ad03aeb55628aeffc2363: Waiting for close lock at 1732506081633Running coprocessor pre-close hooks at 1732506081633Disabling compacts and flushes for region at 1732506081633Disabling writes for close at 1732506081633Writing region close event to WAL at 1732506081634 (+1 ms)Running coprocessor post-close hooks at 1732506081641 (+7 ms)Closed at 1732506081642 (+1 ms) 2024-11-25T03:41:21,642 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,fff,1732506074951.bba1924bf10ad03aeb55628aeffc2363. 2024-11-25T03:41:21,642 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 745356a682ec14d9f6e2066d456cab06, disabling compactions & flushes 2024-11-25T03:41:21,642 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,rrr,1732506074951.745356a682ec14d9f6e2066d456cab06. 2024-11-25T03:41:21,642 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1732506074951.745356a682ec14d9f6e2066d456cab06. 2024-11-25T03:41:21,642 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1732506074951.745356a682ec14d9f6e2066d456cab06. after waiting 0 ms 2024-11-25T03:41:21,643 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1732506074951.745356a682ec14d9f6e2066d456cab06. 2024-11-25T03:41:21,645 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/4c0b379dd00b53bde73bca0e12f321ce/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-25T03:41:21,646 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,iii,1732506074951.4c0b379dd00b53bde73bca0e12f321ce. 2024-11-25T03:41:21,646 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4c0b379dd00b53bde73bca0e12f321ce: Waiting for close lock at 1732506081640Running coprocessor pre-close hooks at 1732506081640Disabling compacts and flushes for region at 1732506081640Disabling writes for close at 1732506081640Writing region close event to WAL at 1732506081641 (+1 ms)Running coprocessor post-close hooks at 1732506081646 (+5 ms)Closed at 1732506081646 2024-11-25T03:41:21,646 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,iii,1732506074951.4c0b379dd00b53bde73bca0e12f321ce. 2024-11-25T03:41:21,646 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4705fa4ae38907dd8b9783be2dff29da, disabling compactions & flushes 2024-11-25T03:41:21,646 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,nnn,1732506074951.4705fa4ae38907dd8b9783be2dff29da. 2024-11-25T03:41:21,646 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1732506074951.4705fa4ae38907dd8b9783be2dff29da. 2024-11-25T03:41:21,646 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1732506074951.4705fa4ae38907dd8b9783be2dff29da. after waiting 0 ms 2024-11-25T03:41:21,646 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1732506074951.4705fa4ae38907dd8b9783be2dff29da. 2024-11-25T03:41:21,651 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/4705fa4ae38907dd8b9783be2dff29da/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-25T03:41:21,652 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,nnn,1732506074951.4705fa4ae38907dd8b9783be2dff29da. 2024-11-25T03:41:21,652 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4705fa4ae38907dd8b9783be2dff29da: Waiting for close lock at 1732506081646Running coprocessor pre-close hooks at 1732506081646Disabling compacts and flushes for region at 1732506081646Disabling writes for close at 1732506081646Writing region close event to WAL at 1732506081647 (+1 ms)Running coprocessor post-close hooks at 1732506081652 (+5 ms)Closed at 1732506081652 2024-11-25T03:41:21,652 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,nnn,1732506074951.4705fa4ae38907dd8b9783be2dff29da. 2024-11-25T03:41:21,652 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0698446036b7ae35179ebc9b07274db5, disabling compactions & flushes 2024-11-25T03:41:21,652 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,sss,1732506074951.0698446036b7ae35179ebc9b07274db5. 2024-11-25T03:41:21,652 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1732506074951.0698446036b7ae35179ebc9b07274db5. 2024-11-25T03:41:21,652 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,sss,1732506074951.0698446036b7ae35179ebc9b07274db5. after waiting 0 ms 2024-11-25T03:41:21,652 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,sss,1732506074951.0698446036b7ae35179ebc9b07274db5. 2024-11-25T03:41:21,658 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/hbase/meta/1588230740/.tmp/info/209813c3bae04c5784b33da0e4a53153 is 138, key is test,ggg,1732506074951.857a6b6999e909a9b73e698c69291205./info:regioninfo/1732506080430/Put/seqid=0 2024-11-25T03:41:21,660 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/745356a682ec14d9f6e2066d456cab06/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-25T03:41:21,660 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/0698446036b7ae35179ebc9b07274db5/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-25T03:41:21,661 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,rrr,1732506074951.745356a682ec14d9f6e2066d456cab06. 2024-11-25T03:41:21,661 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,sss,1732506074951.0698446036b7ae35179ebc9b07274db5. 2024-11-25T03:41:21,661 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0698446036b7ae35179ebc9b07274db5: Waiting for close lock at 1732506081652Running coprocessor pre-close hooks at 1732506081652Disabling compacts and flushes for region at 1732506081652Disabling writes for close at 1732506081652Writing region close event to WAL at 1732506081656 (+4 ms)Running coprocessor post-close hooks at 1732506081661 (+5 ms)Closed at 1732506081661 2024-11-25T03:41:21,661 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 745356a682ec14d9f6e2066d456cab06: Waiting for close lock at 1732506081642Running coprocessor pre-close hooks at 1732506081642Disabling compacts and flushes for region at 1732506081642Disabling writes for close at 1732506081643 (+1 ms)Writing region close event to WAL at 1732506081643Running coprocessor post-close hooks at 1732506081661 (+18 ms)Closed at 1732506081661 2024-11-25T03:41:21,661 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,rrr,1732506074951.745356a682ec14d9f6e2066d456cab06. 2024-11-25T03:41:21,661 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,sss,1732506074951.0698446036b7ae35179ebc9b07274db5. 2024-11-25T03:41:21,661 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 857a6b6999e909a9b73e698c69291205, disabling compactions & flushes 2024-11-25T03:41:21,661 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 85a5fab26752f98e6966cd407e31db69, disabling compactions & flushes 2024-11-25T03:41:21,661 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ggg,1732506074951.857a6b6999e909a9b73e698c69291205. 2024-11-25T03:41:21,661 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ddd,1732506074951.85a5fab26752f98e6966cd407e31db69. 2024-11-25T03:41:21,661 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1732506074951.85a5fab26752f98e6966cd407e31db69. 2024-11-25T03:41:21,661 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1732506074951.857a6b6999e909a9b73e698c69291205. 2024-11-25T03:41:21,661 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1732506074951.85a5fab26752f98e6966cd407e31db69. after waiting 0 ms 2024-11-25T03:41:21,661 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1732506074951.857a6b6999e909a9b73e698c69291205. after waiting 0 ms 2024-11-25T03:41:21,661 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1732506074951.85a5fab26752f98e6966cd407e31db69. 2024-11-25T03:41:21,662 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1732506074951.857a6b6999e909a9b73e698c69291205. 2024-11-25T03:41:21,673 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/85a5fab26752f98e6966cd407e31db69/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-25T03:41:21,674 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ddd,1732506074951.85a5fab26752f98e6966cd407e31db69. 2024-11-25T03:41:21,674 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 85a5fab26752f98e6966cd407e31db69: Waiting for close lock at 1732506081661Running coprocessor pre-close hooks at 1732506081661Disabling compacts and flushes for region at 1732506081661Disabling writes for close at 1732506081661Writing region close event to WAL at 1732506081664 (+3 ms)Running coprocessor post-close hooks at 1732506081674 (+10 ms)Closed at 1732506081674 2024-11-25T03:41:21,674 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ddd,1732506074951.85a5fab26752f98e6966cd407e31db69. 2024-11-25T03:41:21,675 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3b442507eaac3b5222a3c4f97ff80208, disabling compactions & flushes 2024-11-25T03:41:21,675 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,,1732506074951.3b442507eaac3b5222a3c4f97ff80208. 2024-11-25T03:41:21,675 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1732506074951.3b442507eaac3b5222a3c4f97ff80208. 2024-11-25T03:41:21,675 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,,1732506074951.3b442507eaac3b5222a3c4f97ff80208. after waiting 0 ms 2024-11-25T03:41:21,675 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,,1732506074951.3b442507eaac3b5222a3c4f97ff80208. 2024-11-25T03:41:21,676 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/857a6b6999e909a9b73e698c69291205/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-25T03:41:21,677 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ggg,1732506074951.857a6b6999e909a9b73e698c69291205. 2024-11-25T03:41:21,677 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 857a6b6999e909a9b73e698c69291205: Waiting for close lock at 1732506081661Running coprocessor pre-close hooks at 1732506081661Disabling compacts and flushes for region at 1732506081661Disabling writes for close at 1732506081661Writing region close event to WAL at 1732506081665 (+4 ms)Running coprocessor post-close hooks at 1732506081677 (+12 ms)Closed at 1732506081677 2024-11-25T03:41:21,677 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ggg,1732506074951.857a6b6999e909a9b73e698c69291205. 2024-11-25T03:41:21,677 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0c260efb5b2a8a6137010d439efc8d47, disabling compactions & flushes 2024-11-25T03:41:21,677 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,mmm,1732506074951.0c260efb5b2a8a6137010d439efc8d47. 2024-11-25T03:41:21,677 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1732506074951.0c260efb5b2a8a6137010d439efc8d47. 2024-11-25T03:41:21,677 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1732506074951.0c260efb5b2a8a6137010d439efc8d47. after waiting 0 ms 2024-11-25T03:41:21,678 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1732506074951.0c260efb5b2a8a6137010d439efc8d47. 2024-11-25T03:41:21,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741868_1044 (size=8174) 2024-11-25T03:41:21,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41447 is added to blk_1073741868_1044 (size=8174) 2024-11-25T03:41:21,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33637 is added to blk_1073741868_1044 (size=8174) 2024-11-25T03:41:21,680 INFO [regionserver/d73269e94d72:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T03:41:21,680 INFO [regionserver/d73269e94d72:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T03:41:21,680 INFO [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.64 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/hbase/meta/1588230740/.tmp/info/209813c3bae04c5784b33da0e4a53153 2024-11-25T03:41:21,691 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/hbase/meta/1588230740/.tmp/info/209813c3bae04c5784b33da0e4a53153 as hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/hbase/meta/1588230740/info/209813c3bae04c5784b33da0e4a53153 2024-11-25T03:41:21,705 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/3b442507eaac3b5222a3c4f97ff80208/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-25T03:41:21,707 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,,1732506074951.3b442507eaac3b5222a3c4f97ff80208. 2024-11-25T03:41:21,707 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3b442507eaac3b5222a3c4f97ff80208: Waiting for close lock at 1732506081674Running coprocessor pre-close hooks at 1732506081674Disabling compacts and flushes for region at 1732506081674Disabling writes for close at 1732506081675 (+1 ms)Writing region close event to WAL at 1732506081692 (+17 ms)Running coprocessor post-close hooks at 1732506081707 (+15 ms)Closed at 1732506081707 2024-11-25T03:41:21,707 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,,1732506074951.3b442507eaac3b5222a3c4f97ff80208. 2024-11-25T03:41:21,707 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f5f64cb2042c701322ac00e312f65f16, disabling compactions & flushes 2024-11-25T03:41:21,707 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,jjj,1732506074951.f5f64cb2042c701322ac00e312f65f16. 2024-11-25T03:41:21,707 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1732506074951.f5f64cb2042c701322ac00e312f65f16. 2024-11-25T03:41:21,707 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1732506074951.f5f64cb2042c701322ac00e312f65f16. after waiting 0 ms 2024-11-25T03:41:21,707 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1732506074951.f5f64cb2042c701322ac00e312f65f16. 2024-11-25T03:41:21,709 INFO [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/hbase/meta/1588230740/info/209813c3bae04c5784b33da0e4a53153, entries=24, sequenceid=75, filesize=8.0 K 2024-11-25T03:41:21,711 INFO [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.64 KB/2705, heapSize ~4.66 KB/4768, currentSize=0 B/0 for 1588230740 in 76ms, sequenceid=75, compaction requested=false 2024-11-25T03:41:21,738 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/0c260efb5b2a8a6137010d439efc8d47/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-25T03:41:21,738 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,mmm,1732506074951.0c260efb5b2a8a6137010d439efc8d47. 2024-11-25T03:41:21,738 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0c260efb5b2a8a6137010d439efc8d47: Waiting for close lock at 1732506081677Running coprocessor pre-close hooks at 1732506081677Disabling compacts and flushes for region at 1732506081677Disabling writes for close at 1732506081678 (+1 ms)Writing region close event to WAL at 1732506081712 (+34 ms)Running coprocessor post-close hooks at 1732506081738 (+26 ms)Closed at 1732506081738 2024-11-25T03:41:21,739 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,mmm,1732506074951.0c260efb5b2a8a6137010d439efc8d47. 2024-11-25T03:41:21,739 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5e3a490d70b339ac73875ea7638f934f, disabling compactions & flushes 2024-11-25T03:41:21,739 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ccc,1732506074951.5e3a490d70b339ac73875ea7638f934f. 2024-11-25T03:41:21,739 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1732506074951.5e3a490d70b339ac73875ea7638f934f. 2024-11-25T03:41:21,739 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1732506074951.5e3a490d70b339ac73875ea7638f934f. after waiting 0 ms 2024-11-25T03:41:21,739 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1732506074951.5e3a490d70b339ac73875ea7638f934f. 2024-11-25T03:41:21,741 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/f5f64cb2042c701322ac00e312f65f16/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-25T03:41:21,742 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,jjj,1732506074951.f5f64cb2042c701322ac00e312f65f16. 2024-11-25T03:41:21,743 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f5f64cb2042c701322ac00e312f65f16: Waiting for close lock at 1732506081707Running coprocessor pre-close hooks at 1732506081707Disabling compacts and flushes for region at 1732506081707Disabling writes for close at 1732506081707Writing region close event to WAL at 1732506081713 (+6 ms)Running coprocessor post-close hooks at 1732506081742 (+29 ms)Closed at 1732506081742 2024-11-25T03:41:21,743 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,jjj,1732506074951.f5f64cb2042c701322ac00e312f65f16. 2024-11-25T03:41:21,743 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5da71c9c32aa33cf6f067ea6d82ae84f, disabling compactions & flushes 2024-11-25T03:41:21,743 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ooo,1732506074951.5da71c9c32aa33cf6f067ea6d82ae84f. 2024-11-25T03:41:21,743 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1732506074951.5da71c9c32aa33cf6f067ea6d82ae84f. 2024-11-25T03:41:21,743 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1732506074951.5da71c9c32aa33cf6f067ea6d82ae84f. after waiting 0 ms 2024-11-25T03:41:21,743 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1732506074951.5da71c9c32aa33cf6f067ea6d82ae84f. 2024-11-25T03:41:21,754 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/5e3a490d70b339ac73875ea7638f934f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-25T03:41:21,755 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ccc,1732506074951.5e3a490d70b339ac73875ea7638f934f. 2024-11-25T03:41:21,755 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5e3a490d70b339ac73875ea7638f934f: Waiting for close lock at 1732506081739Running coprocessor pre-close hooks at 1732506081739Disabling compacts and flushes for region at 1732506081739Disabling writes for close at 1732506081739Writing region close event to WAL at 1732506081744 (+5 ms)Running coprocessor post-close hooks at 1732506081755 (+11 ms)Closed at 1732506081755 2024-11-25T03:41:21,755 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ccc,1732506074951.5e3a490d70b339ac73875ea7638f934f. 2024-11-25T03:41:21,756 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8a7d4164efdaed87f07b6f76a94819b3, disabling compactions & flushes 2024-11-25T03:41:21,756 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,eee,1732506074951.8a7d4164efdaed87f07b6f76a94819b3. 2024-11-25T03:41:21,756 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1732506074951.8a7d4164efdaed87f07b6f76a94819b3. 2024-11-25T03:41:21,756 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,eee,1732506074951.8a7d4164efdaed87f07b6f76a94819b3. after waiting 0 ms 2024-11-25T03:41:21,756 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,eee,1732506074951.8a7d4164efdaed87f07b6f76a94819b3. 2024-11-25T03:41:21,758 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/5da71c9c32aa33cf6f067ea6d82ae84f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-25T03:41:21,760 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ooo,1732506074951.5da71c9c32aa33cf6f067ea6d82ae84f. 2024-11-25T03:41:21,760 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5da71c9c32aa33cf6f067ea6d82ae84f: Waiting for close lock at 1732506081743Running coprocessor pre-close hooks at 1732506081743Disabling compacts and flushes for region at 1732506081743Disabling writes for close at 1732506081743Writing region close event to WAL at 1732506081748 (+5 ms)Running coprocessor post-close hooks at 1732506081760 (+12 ms)Closed at 1732506081760 2024-11-25T03:41:21,760 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ooo,1732506074951.5da71c9c32aa33cf6f067ea6d82ae84f. 2024-11-25T03:41:21,760 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2b224c7c31e6ff664ce67ac884f573d7, disabling compactions & flushes 2024-11-25T03:41:21,761 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,www,1732506074951.2b224c7c31e6ff664ce67ac884f573d7. 2024-11-25T03:41:21,761 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1732506074951.2b224c7c31e6ff664ce67ac884f573d7. 2024-11-25T03:41:21,761 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,www,1732506074951.2b224c7c31e6ff664ce67ac884f573d7. after waiting 0 ms 2024-11-25T03:41:21,761 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,www,1732506074951.2b224c7c31e6ff664ce67ac884f573d7. 2024-11-25T03:41:21,782 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/hbase/meta/1588230740/recovered.edits/78.seqid, newMaxSeqId=78, maxSeqId=64 2024-11-25T03:41:21,784 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-25T03:41:21,784 INFO [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-25T03:41:21,784 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732506081634Running coprocessor pre-close hooks at 1732506081634Disabling compacts and flushes for region at 1732506081634Disabling writes for close at 1732506081635 (+1 ms)Obtaining lock to block concurrent updates at 1732506081635Preparing flush snapshotting stores in 1588230740 at 1732506081635Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=2705, getHeapSize=5488, getOffHeapSize=0, getCellsCount=24 at 1732506081635Flushing stores of hbase:meta,,1.1588230740 at 1732506081636 (+1 ms)Flushing 1588230740/info: creating writer at 1732506081636Flushing 1588230740/info: appending metadata at 1732506081658 (+22 ms)Flushing 1588230740/info: closing flushed file at 1732506081658Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@442a0ca5: reopening flushed file at 1732506081690 (+32 ms)Finished flush of dataSize ~2.64 KB/2705, heapSize ~4.66 KB/4768, currentSize=0 B/0 for 1588230740 in 76ms, sequenceid=75, compaction requested=false at 1732506081711 (+21 ms)Writing region close event to WAL at 1732506081752 (+41 ms)Running coprocessor post-close hooks at 1732506081784 (+32 ms)Closed at 1732506081784 2024-11-25T03:41:21,784 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-25T03:41:21,785 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/8a7d4164efdaed87f07b6f76a94819b3/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-25T03:41:21,787 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,eee,1732506074951.8a7d4164efdaed87f07b6f76a94819b3. 2024-11-25T03:41:21,787 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8a7d4164efdaed87f07b6f76a94819b3: Waiting for close lock at 1732506081756Running coprocessor pre-close hooks at 1732506081756Disabling compacts and flushes for region at 1732506081756Disabling writes for close at 1732506081756Writing region close event to WAL at 1732506081774 (+18 ms)Running coprocessor post-close hooks at 1732506081786 (+12 ms)Closed at 1732506081787 (+1 ms) 2024-11-25T03:41:21,787 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,eee,1732506074951.8a7d4164efdaed87f07b6f76a94819b3. 2024-11-25T03:41:21,787 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 706facb174cc9547b71f08aa672ddcee, disabling compactions & flushes 2024-11-25T03:41:21,787 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,vvv,1732506074951.706facb174cc9547b71f08aa672ddcee. 2024-11-25T03:41:21,787 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1732506074951.706facb174cc9547b71f08aa672ddcee. 2024-11-25T03:41:21,787 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1732506074951.706facb174cc9547b71f08aa672ddcee. after waiting 0 ms 2024-11-25T03:41:21,788 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1732506074951.706facb174cc9547b71f08aa672ddcee. 2024-11-25T03:41:21,793 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/2b224c7c31e6ff664ce67ac884f573d7/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-25T03:41:21,794 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,www,1732506074951.2b224c7c31e6ff664ce67ac884f573d7. 2024-11-25T03:41:21,794 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2b224c7c31e6ff664ce67ac884f573d7: Waiting for close lock at 1732506081760Running coprocessor pre-close hooks at 1732506081760Disabling compacts and flushes for region at 1732506081760Disabling writes for close at 1732506081761 (+1 ms)Writing region close event to WAL at 1732506081782 (+21 ms)Running coprocessor post-close hooks at 1732506081794 (+12 ms)Closed at 1732506081794 2024-11-25T03:41:21,795 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,www,1732506074951.2b224c7c31e6ff664ce67ac884f573d7. 2024-11-25T03:41:21,795 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5e791add66a4fee50be34aa83a6d502a, disabling compactions & flushes 2024-11-25T03:41:21,795 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,hhh,1732506074951.5e791add66a4fee50be34aa83a6d502a. 2024-11-25T03:41:21,795 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1732506074951.5e791add66a4fee50be34aa83a6d502a. 2024-11-25T03:41:21,795 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1732506074951.5e791add66a4fee50be34aa83a6d502a. after waiting 0 ms 2024-11-25T03:41:21,795 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1732506074951.5e791add66a4fee50be34aa83a6d502a. 2024-11-25T03:41:21,800 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/706facb174cc9547b71f08aa672ddcee/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-25T03:41:21,802 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,vvv,1732506074951.706facb174cc9547b71f08aa672ddcee. 2024-11-25T03:41:21,802 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 706facb174cc9547b71f08aa672ddcee: Waiting for close lock at 1732506081787Running coprocessor pre-close hooks at 1732506081787Disabling compacts and flushes for region at 1732506081787Disabling writes for close at 1732506081787Writing region close event to WAL at 1732506081792 (+5 ms)Running coprocessor post-close hooks at 1732506081802 (+10 ms)Closed at 1732506081802 2024-11-25T03:41:21,802 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,vvv,1732506074951.706facb174cc9547b71f08aa672ddcee. 2024-11-25T03:41:21,803 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8b5ae4b02f22b24f40505bb660957004, disabling compactions & flushes 2024-11-25T03:41:21,803 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ttt,1732506074951.8b5ae4b02f22b24f40505bb660957004. 2024-11-25T03:41:21,804 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1732506074951.8b5ae4b02f22b24f40505bb660957004. 2024-11-25T03:41:21,804 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1732506074951.8b5ae4b02f22b24f40505bb660957004. after waiting 0 ms 2024-11-25T03:41:21,804 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1732506074951.8b5ae4b02f22b24f40505bb660957004. 2024-11-25T03:41:21,804 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/5e791add66a4fee50be34aa83a6d502a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-25T03:41:21,805 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,hhh,1732506074951.5e791add66a4fee50be34aa83a6d502a. 2024-11-25T03:41:21,805 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5e791add66a4fee50be34aa83a6d502a: Waiting for close lock at 1732506081795Running coprocessor pre-close hooks at 1732506081795Disabling compacts and flushes for region at 1732506081795Disabling writes for close at 1732506081795Writing region close event to WAL at 1732506081799 (+4 ms)Running coprocessor post-close hooks at 1732506081805 (+6 ms)Closed at 1732506081805 2024-11-25T03:41:21,806 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,hhh,1732506074951.5e791add66a4fee50be34aa83a6d502a. 2024-11-25T03:41:21,806 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6aed1980a8b63b44c776d0990f923aa7, disabling compactions & flushes 2024-11-25T03:41:21,806 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,uuu,1732506074951.6aed1980a8b63b44c776d0990f923aa7. 2024-11-25T03:41:21,806 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1732506074951.6aed1980a8b63b44c776d0990f923aa7. 2024-11-25T03:41:21,806 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1732506074951.6aed1980a8b63b44c776d0990f923aa7. after waiting 0 ms 2024-11-25T03:41:21,806 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1732506074951.6aed1980a8b63b44c776d0990f923aa7. 2024-11-25T03:41:21,822 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/8b5ae4b02f22b24f40505bb660957004/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-25T03:41:21,830 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/6aed1980a8b63b44c776d0990f923aa7/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-25T03:41:21,830 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ttt,1732506074951.8b5ae4b02f22b24f40505bb660957004. 2024-11-25T03:41:21,830 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8b5ae4b02f22b24f40505bb660957004: Waiting for close lock at 1732506081803Running coprocessor pre-close hooks at 1732506081803Disabling compacts and flushes for region at 1732506081803Disabling writes for close at 1732506081804 (+1 ms)Writing region close event to WAL at 1732506081812 (+8 ms)Running coprocessor post-close hooks at 1732506081830 (+18 ms)Closed at 1732506081830 2024-11-25T03:41:21,830 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ttt,1732506074951.8b5ae4b02f22b24f40505bb660957004. 2024-11-25T03:41:21,831 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c769ca025a015554efcc98619008be96, disabling compactions & flushes 2024-11-25T03:41:21,831 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,zzz,1732506074951.c769ca025a015554efcc98619008be96. 2024-11-25T03:41:21,831 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1732506074951.c769ca025a015554efcc98619008be96. 2024-11-25T03:41:21,831 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1732506074951.c769ca025a015554efcc98619008be96. after waiting 0 ms 2024-11-25T03:41:21,831 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1732506074951.c769ca025a015554efcc98619008be96. 2024-11-25T03:41:21,831 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,uuu,1732506074951.6aed1980a8b63b44c776d0990f923aa7. 2024-11-25T03:41:21,831 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6aed1980a8b63b44c776d0990f923aa7: Waiting for close lock at 1732506081806Running coprocessor pre-close hooks at 1732506081806Disabling compacts and flushes for region at 1732506081806Disabling writes for close at 1732506081806Writing region close event to WAL at 1732506081814 (+8 ms)Running coprocessor post-close hooks at 1732506081831 (+17 ms)Closed at 1732506081831 2024-11-25T03:41:21,832 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,uuu,1732506074951.6aed1980a8b63b44c776d0990f923aa7. 2024-11-25T03:41:21,832 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5f812dacea03120e7730979dc37e38c6, disabling compactions & flushes 2024-11-25T03:41:21,832 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,xxx,1732506074951.5f812dacea03120e7730979dc37e38c6. 2024-11-25T03:41:21,832 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1732506074951.5f812dacea03120e7730979dc37e38c6. 2024-11-25T03:41:21,832 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1732506074951.5f812dacea03120e7730979dc37e38c6. after waiting 0 ms 2024-11-25T03:41:21,832 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1732506074951.5f812dacea03120e7730979dc37e38c6. 2024-11-25T03:41:21,834 DEBUG [RS:2;d73269e94d72:38397 {}] regionserver.HRegionServer(1351): Waiting on 2fc93e882c2b7c5acdb53e9cf1071dc8, c769ca025a015554efcc98619008be96, eb9ff3be6d639a626d88387efa9120fd 2024-11-25T03:41:21,835 DEBUG [RS:1;d73269e94d72:33145 {}] regionserver.HRegionServer(1351): Waiting on 5f812dacea03120e7730979dc37e38c6, 922ebf14d68aba40da3a2e99777c777b, a4b8682bf62ac734aa4174eba4ee1ddc, c73ee0e3b504288792edda3ec9a26049 2024-11-25T03:41:21,858 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/c769ca025a015554efcc98619008be96/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-25T03:41:21,859 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,zzz,1732506074951.c769ca025a015554efcc98619008be96. 2024-11-25T03:41:21,859 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c769ca025a015554efcc98619008be96: Waiting for close lock at 1732506081831Running coprocessor pre-close hooks at 1732506081831Disabling compacts and flushes for region at 1732506081831Disabling writes for close at 1732506081831Writing region close event to WAL at 1732506081832 (+1 ms)Running coprocessor post-close hooks at 1732506081859 (+27 ms)Closed at 1732506081859 2024-11-25T03:41:21,860 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,zzz,1732506074951.c769ca025a015554efcc98619008be96. 2024-11-25T03:41:21,860 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing eb9ff3be6d639a626d88387efa9120fd, disabling compactions & flushes 2024-11-25T03:41:21,860 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,lll,1732506074951.eb9ff3be6d639a626d88387efa9120fd. 2024-11-25T03:41:21,860 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1732506074951.eb9ff3be6d639a626d88387efa9120fd. 2024-11-25T03:41:21,860 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,lll,1732506074951.eb9ff3be6d639a626d88387efa9120fd. after waiting 0 ms 2024-11-25T03:41:21,860 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,lll,1732506074951.eb9ff3be6d639a626d88387efa9120fd. 2024-11-25T03:41:21,876 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/5f812dacea03120e7730979dc37e38c6/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-25T03:41:21,877 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,xxx,1732506074951.5f812dacea03120e7730979dc37e38c6. 2024-11-25T03:41:21,877 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/eb9ff3be6d639a626d88387efa9120fd/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-25T03:41:21,877 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5f812dacea03120e7730979dc37e38c6: Waiting for close lock at 1732506081832Running coprocessor pre-close hooks at 1732506081832Disabling compacts and flushes for region at 1732506081832Disabling writes for close at 1732506081832Writing region close event to WAL at 1732506081836 (+4 ms)Running coprocessor post-close hooks at 1732506081877 (+41 ms)Closed at 1732506081877 2024-11-25T03:41:21,877 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,xxx,1732506074951.5f812dacea03120e7730979dc37e38c6. 2024-11-25T03:41:21,878 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a4b8682bf62ac734aa4174eba4ee1ddc, disabling compactions & flushes 2024-11-25T03:41:21,878 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,kkk,1732506074951.a4b8682bf62ac734aa4174eba4ee1ddc. 2024-11-25T03:41:21,878 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1732506074951.a4b8682bf62ac734aa4174eba4ee1ddc. 2024-11-25T03:41:21,878 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1732506074951.a4b8682bf62ac734aa4174eba4ee1ddc. after waiting 0 ms 2024-11-25T03:41:21,878 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1732506074951.a4b8682bf62ac734aa4174eba4ee1ddc. 2024-11-25T03:41:21,878 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,lll,1732506074951.eb9ff3be6d639a626d88387efa9120fd. 2024-11-25T03:41:21,878 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for eb9ff3be6d639a626d88387efa9120fd: Waiting for close lock at 1732506081860Running coprocessor pre-close hooks at 1732506081860Disabling compacts and flushes for region at 1732506081860Disabling writes for close at 1732506081860Writing region close event to WAL at 1732506081872 (+12 ms)Running coprocessor post-close hooks at 1732506081878 (+6 ms)Closed at 1732506081878 2024-11-25T03:41:21,878 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,lll,1732506074951.eb9ff3be6d639a626d88387efa9120fd. 2024-11-25T03:41:21,878 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2fc93e882c2b7c5acdb53e9cf1071dc8, disabling compactions & flushes 2024-11-25T03:41:21,879 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ppp,1732506074951.2fc93e882c2b7c5acdb53e9cf1071dc8. 2024-11-25T03:41:21,879 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1732506074951.2fc93e882c2b7c5acdb53e9cf1071dc8. 2024-11-25T03:41:21,879 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1732506074951.2fc93e882c2b7c5acdb53e9cf1071dc8. after waiting 0 ms 2024-11-25T03:41:21,879 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1732506074951.2fc93e882c2b7c5acdb53e9cf1071dc8. 2024-11-25T03:41:21,891 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/a4b8682bf62ac734aa4174eba4ee1ddc/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-25T03:41:21,892 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,kkk,1732506074951.a4b8682bf62ac734aa4174eba4ee1ddc. 2024-11-25T03:41:21,892 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a4b8682bf62ac734aa4174eba4ee1ddc: Waiting for close lock at 1732506081878Running coprocessor pre-close hooks at 1732506081878Disabling compacts and flushes for region at 1732506081878Disabling writes for close at 1732506081878Writing region close event to WAL at 1732506081879 (+1 ms)Running coprocessor post-close hooks at 1732506081892 (+13 ms)Closed at 1732506081892 2024-11-25T03:41:21,892 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,kkk,1732506074951.a4b8682bf62ac734aa4174eba4ee1ddc. 2024-11-25T03:41:21,892 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 922ebf14d68aba40da3a2e99777c777b, disabling compactions & flushes 2024-11-25T03:41:21,892 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,qqq,1732506074951.922ebf14d68aba40da3a2e99777c777b. 2024-11-25T03:41:21,892 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1732506074951.922ebf14d68aba40da3a2e99777c777b. 2024-11-25T03:41:21,893 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1732506074951.922ebf14d68aba40da3a2e99777c777b. after waiting 0 ms 2024-11-25T03:41:21,893 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1732506074951.922ebf14d68aba40da3a2e99777c777b. 2024-11-25T03:41:21,899 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/2fc93e882c2b7c5acdb53e9cf1071dc8/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-25T03:41:21,900 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ppp,1732506074951.2fc93e882c2b7c5acdb53e9cf1071dc8. 2024-11-25T03:41:21,900 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2fc93e882c2b7c5acdb53e9cf1071dc8: Waiting for close lock at 1732506081878Running coprocessor pre-close hooks at 1732506081878Disabling compacts and flushes for region at 1732506081878Disabling writes for close at 1732506081879 (+1 ms)Writing region close event to WAL at 1732506081880 (+1 ms)Running coprocessor post-close hooks at 1732506081900 (+20 ms)Closed at 1732506081900 2024-11-25T03:41:21,900 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ppp,1732506074951.2fc93e882c2b7c5acdb53e9cf1071dc8. 2024-11-25T03:41:21,903 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/922ebf14d68aba40da3a2e99777c777b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-25T03:41:21,904 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,qqq,1732506074951.922ebf14d68aba40da3a2e99777c777b. 2024-11-25T03:41:21,904 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 922ebf14d68aba40da3a2e99777c777b: Waiting for close lock at 1732506081892Running coprocessor pre-close hooks at 1732506081892Disabling compacts and flushes for region at 1732506081892Disabling writes for close at 1732506081893 (+1 ms)Writing region close event to WAL at 1732506081899 (+6 ms)Running coprocessor post-close hooks at 1732506081904 (+5 ms)Closed at 1732506081904 2024-11-25T03:41:21,904 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,qqq,1732506074951.922ebf14d68aba40da3a2e99777c777b. 2024-11-25T03:41:21,904 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c73ee0e3b504288792edda3ec9a26049, disabling compactions & flushes 2024-11-25T03:41:21,904 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,bbb,1732506074951.c73ee0e3b504288792edda3ec9a26049. 2024-11-25T03:41:21,904 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1732506074951.c73ee0e3b504288792edda3ec9a26049. 2024-11-25T03:41:21,904 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1732506074951.c73ee0e3b504288792edda3ec9a26049. after waiting 0 ms 2024-11-25T03:41:21,904 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1732506074951.c73ee0e3b504288792edda3ec9a26049. 2024-11-25T03:41:21,915 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/data/default/test/c73ee0e3b504288792edda3ec9a26049/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-25T03:41:21,916 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,bbb,1732506074951.c73ee0e3b504288792edda3ec9a26049. 2024-11-25T03:41:21,916 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c73ee0e3b504288792edda3ec9a26049: Waiting for close lock at 1732506081904Running coprocessor pre-close hooks at 1732506081904Disabling compacts and flushes for region at 1732506081904Disabling writes for close at 1732506081904Writing region close event to WAL at 1732506081908 (+4 ms)Running coprocessor post-close hooks at 1732506081916 (+8 ms)Closed at 1732506081916 2024-11-25T03:41:21,916 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,bbb,1732506074951.c73ee0e3b504288792edda3ec9a26049. 2024-11-25T03:41:21,957 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-25T03:41:21,958 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-25T03:41:21,958 INFO [regionserver/d73269e94d72:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-25T03:41:21,958 INFO [regionserver/d73269e94d72:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-25T03:41:21,960 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test 2024-11-25T03:41:21,960 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test Metrics about Tables on a single HBase RegionServer 2024-11-25T03:41:21,962 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-25T03:41:21,962 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-25T03:41:21,962 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-25T03:41:21,962 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-25T03:41:21,965 INFO [regionserver/d73269e94d72:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-25T03:41:21,966 INFO [regionserver/d73269e94d72:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-25T03:41:22,034 INFO [RS:2;d73269e94d72:38397 {}] regionserver.HRegionServer(976): stopping server d73269e94d72,38397,1732506072528; all regions closed. 2024-11-25T03:41:22,035 INFO [RS:1;d73269e94d72:33145 {}] regionserver.HRegionServer(976): stopping server d73269e94d72,33145,1732506072477; all regions closed. 2024-11-25T03:41:22,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741835_1011 (size=7074) 2024-11-25T03:41:22,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33637 is added to blk_1073741835_1011 (size=7074) 2024-11-25T03:41:22,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33637 is added to blk_1073741836_1012 (size=36942) 2024-11-25T03:41:22,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741836_1012 (size=36942) 2024-11-25T03:41:22,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41447 is added to blk_1073741835_1011 (size=7074) 2024-11-25T03:41:22,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41447 is added to blk_1073741836_1012 (size=36942) 2024-11-25T03:41:22,043 DEBUG [RS:2;d73269e94d72:38397 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/oldWALs 2024-11-25T03:41:22,043 DEBUG [RS:1;d73269e94d72:33145 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/oldWALs 2024-11-25T03:41:22,043 INFO [RS:2;d73269e94d72:38397 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL d73269e94d72%2C38397%2C1732506072528:(num 1732506074123) 2024-11-25T03:41:22,043 INFO [RS:1;d73269e94d72:33145 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL d73269e94d72%2C33145%2C1732506072477.meta:.meta(num 1732506074505) 2024-11-25T03:41:22,043 DEBUG [RS:2;d73269e94d72:38397 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:41:22,043 INFO [RS:2;d73269e94d72:38397 {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T03:41:22,044 INFO [RS:2;d73269e94d72:38397 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T03:41:22,044 INFO [RS:2;d73269e94d72:38397 {}] hbase.ChoreService(370): Chore service for: regionserver/d73269e94d72:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-25T03:41:22,044 INFO [RS:2;d73269e94d72:38397 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-25T03:41:22,044 INFO [RS:2;d73269e94d72:38397 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-25T03:41:22,044 INFO [RS:2;d73269e94d72:38397 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-25T03:41:22,044 INFO [RS:2;d73269e94d72:38397 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T03:41:22,044 INFO [RS:2;d73269e94d72:38397 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38397 2024-11-25T03:41:22,045 INFO [regionserver/d73269e94d72:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-25T03:41:22,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41447 is added to blk_1073741834_1010 (size=7064) 2024-11-25T03:41:22,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33637 is added to blk_1073741834_1010 (size=7064) 2024-11-25T03:41:22,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741834_1010 (size=7064) 2024-11-25T03:41:22,056 DEBUG [RS:1;d73269e94d72:33145 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/oldWALs 2024-11-25T03:41:22,056 INFO [RS:1;d73269e94d72:33145 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL d73269e94d72%2C33145%2C1732506072477:(num 1732506074123) 2024-11-25T03:41:22,056 DEBUG [RS:1;d73269e94d72:33145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:41:22,056 INFO [RS:1;d73269e94d72:33145 {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T03:41:22,056 INFO [RS:1;d73269e94d72:33145 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T03:41:22,057 INFO [RS:1;d73269e94d72:33145 {}] hbase.ChoreService(370): Chore service for: regionserver/d73269e94d72:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-25T03:41:22,057 INFO [RS:1;d73269e94d72:33145 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T03:41:22,057 INFO [regionserver/d73269e94d72:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-25T03:41:22,057 INFO [RS:1;d73269e94d72:33145 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33145 2024-11-25T03:41:22,058 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40325-0x1017014aa590000, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T03:41:22,058 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38397-0x1017014aa590003, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d73269e94d72,38397,1732506072528 2024-11-25T03:41:22,058 INFO [RS:2;d73269e94d72:38397 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T03:41:22,068 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33145-0x1017014aa590002, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d73269e94d72,33145,1732506072477 2024-11-25T03:41:22,068 INFO [RS:1;d73269e94d72:33145 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T03:41:22,068 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d73269e94d72,38397,1732506072528] 2024-11-25T03:41:22,089 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/d73269e94d72,38397,1732506072528 already deleted, retry=false 2024-11-25T03:41:22,089 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; d73269e94d72,38397,1732506072528 expired; onlineServers=1 2024-11-25T03:41:22,089 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d73269e94d72,33145,1732506072477] 2024-11-25T03:41:22,100 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/d73269e94d72,33145,1732506072477 already deleted, retry=false 2024-11-25T03:41:22,100 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; d73269e94d72,33145,1732506072477 expired; onlineServers=0 2024-11-25T03:41:22,100 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'd73269e94d72,40325,1732506071573' ***** 2024-11-25T03:41:22,100 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-25T03:41:22,100 INFO [M:0;d73269e94d72:40325 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T03:41:22,100 INFO [M:0;d73269e94d72:40325 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T03:41:22,100 DEBUG [M:0;d73269e94d72:40325 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-25T03:41:22,101 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-25T03:41:22,101 DEBUG [M:0;d73269e94d72:40325 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-25T03:41:22,101 DEBUG [master/d73269e94d72:0:becomeActiveMaster-HFileCleaner.small.0-1732506073676 {}] cleaner.HFileCleaner(306): Exit Thread[master/d73269e94d72:0:becomeActiveMaster-HFileCleaner.small.0-1732506073676,5,FailOnTimeoutGroup] 2024-11-25T03:41:22,101 DEBUG [master/d73269e94d72:0:becomeActiveMaster-HFileCleaner.large.0-1732506073674 {}] cleaner.HFileCleaner(306): Exit Thread[master/d73269e94d72:0:becomeActiveMaster-HFileCleaner.large.0-1732506073674,5,FailOnTimeoutGroup] 2024-11-25T03:41:22,101 INFO [M:0;d73269e94d72:40325 {}] hbase.ChoreService(370): Chore service for: master/d73269e94d72:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-25T03:41:22,101 INFO [M:0;d73269e94d72:40325 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T03:41:22,101 DEBUG [M:0;d73269e94d72:40325 {}] master.HMaster(1795): Stopping service threads 2024-11-25T03:41:22,101 INFO [M:0;d73269e94d72:40325 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-25T03:41:22,101 INFO [M:0;d73269e94d72:40325 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T03:41:22,102 INFO [M:0;d73269e94d72:40325 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-25T03:41:22,102 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-25T03:41:22,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40325-0x1017014aa590000, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-25T03:41:22,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40325-0x1017014aa590000, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:41:22,113 DEBUG [M:0;d73269e94d72:40325 {}] zookeeper.ZKUtil(347): master:40325-0x1017014aa590000, quorum=127.0.0.1:56467, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-25T03:41:22,114 WARN [M:0;d73269e94d72:40325 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-25T03:41:22,114 INFO [M:0;d73269e94d72:40325 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/.lastflushedseqids 2024-11-25T03:41:22,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33637 is added to blk_1073741869_1045 (size=1) 2024-11-25T03:41:22,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741869_1045 (size=1) 2024-11-25T03:41:22,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41447 is added to blk_1073741869_1045 (size=1) 2024-11-25T03:41:22,125 INFO [M:0;d73269e94d72:40325 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-25T03:41:22,126 INFO [M:0;d73269e94d72:40325 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-25T03:41:22,126 DEBUG [M:0;d73269e94d72:40325 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-25T03:41:22,126 INFO [M:0;d73269e94d72:40325 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T03:41:22,126 DEBUG [M:0;d73269e94d72:40325 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T03:41:22,126 DEBUG [M:0;d73269e94d72:40325 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-25T03:41:22,126 DEBUG [M:0;d73269e94d72:40325 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T03:41:22,126 INFO [M:0;d73269e94d72:40325 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=259.82 KB heapSize=316.67 KB 2024-11-25T03:41:22,141 DEBUG [M:0;d73269e94d72:40325 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b8497d83ea8b462ebc936a6d6a3e7af1 is 82, key is hbase:meta,,1/info:regioninfo/1732506080010/Put/seqid=0 2024-11-25T03:41:22,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41447 is added to blk_1073741870_1046 (size=6342) 2024-11-25T03:41:22,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33637 is added to blk_1073741870_1046 (size=6342) 2024-11-25T03:41:22,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741870_1046 (size=6342) 2024-11-25T03:41:22,148 INFO [M:0;d73269e94d72:40325 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.79 KB at sequenceid=603 (bloomFilter=true), to=hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b8497d83ea8b462ebc936a6d6a3e7af1 2024-11-25T03:41:22,173 DEBUG [M:0;d73269e94d72:40325 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44131/user/jenkins/test-data/148b7cf6-7804-0fe7-17e0-1abad7dbe921/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a52dcc5fe8ce434f8b4881d96cf0549f is 2984, key is \x00\x00\x00\x00\x00\x00\x00