2024-11-15 03:13:33,701 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-15 03:13:33,717 main DEBUG Took 0.013625 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-15 03:13:33,717 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-15 03:13:33,718 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-15 03:13:33,719 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-15 03:13:33,720 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 03:13:33,728 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-15 03:13:33,742 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 03:13:33,743 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 03:13:33,744 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 03:13:33,745 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 03:13:33,746 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 03:13:33,746 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 03:13:33,748 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 03:13:33,748 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 03:13:33,749 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 03:13:33,749 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 03:13:33,750 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 03:13:33,751 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 03:13:33,751 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 03:13:33,752 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 03:13:33,753 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 03:13:33,753 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 03:13:33,754 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 03:13:33,754 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 03:13:33,755 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 03:13:33,755 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 03:13:33,756 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 03:13:33,756 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 03:13:33,757 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 03:13:33,757 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 03:13:33,758 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 03:13:33,758 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-15 03:13:33,760 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 03:13:33,762 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-15 03:13:33,765 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-15 03:13:33,765 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-15 03:13:33,767 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-15 03:13:33,768 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-15 03:13:33,779 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-15 03:13:33,782 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-15 03:13:33,784 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-15 03:13:33,785 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-15 03:13:33,785 main DEBUG createAppenders(={Console}) 2024-11-15 03:13:33,786 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 initialized 2024-11-15 03:13:33,787 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-15 03:13:33,787 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 OK. 2024-11-15 03:13:33,788 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-15 03:13:33,788 main DEBUG OutputStream closed 2024-11-15 03:13:33,789 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-15 03:13:33,789 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-15 03:13:33,790 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@c88a337 OK 2024-11-15 03:13:33,877 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-15 03:13:33,879 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-15 03:13:33,880 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-15 03:13:33,881 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-15 03:13:33,882 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-15 03:13:33,882 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-15 03:13:33,883 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-15 03:13:33,883 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-15 03:13:33,883 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-15 03:13:33,884 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-15 03:13:33,884 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-15 03:13:33,884 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-15 03:13:33,885 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-15 03:13:33,885 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-15 03:13:33,885 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-15 03:13:33,886 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-15 03:13:33,886 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-15 03:13:33,887 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-15 03:13:33,889 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-15 03:13:33,890 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@13330ac6) with optional ClassLoader: null 2024-11-15 03:13:33,890 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-15 03:13:33,891 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@13330ac6] started OK. 2024-11-15T03:13:33,903 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP timeout: 13 mins 2024-11-15 03:13:33,906 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-15 03:13:33,906 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-15T03:13:34,131 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/c769a833-6561-0e53-79a7-5504d78fc044 2024-11-15T03:13:34,150 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$HMasterForTest, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-15T03:13:34,170 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/c769a833-6561-0e53-79a7-5504d78fc044/cluster_03df53d9-c732-dbe9-3abf-bcf77ad8c819, deleteOnExit=true 2024-11-15T03:13:34,170 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-15T03:13:34,172 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/c769a833-6561-0e53-79a7-5504d78fc044/test.cache.data in system properties and HBase conf 2024-11-15T03:13:34,173 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/c769a833-6561-0e53-79a7-5504d78fc044/hadoop.tmp.dir in system properties and HBase conf 2024-11-15T03:13:34,174 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/c769a833-6561-0e53-79a7-5504d78fc044/hadoop.log.dir in system properties and HBase conf 2024-11-15T03:13:34,175 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/c769a833-6561-0e53-79a7-5504d78fc044/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-15T03:13:34,176 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/c769a833-6561-0e53-79a7-5504d78fc044/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-15T03:13:34,176 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-15T03:13:34,270 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-15T03:13:34,356 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-15T03:13:34,360 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/c769a833-6561-0e53-79a7-5504d78fc044/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-15T03:13:34,360 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/c769a833-6561-0e53-79a7-5504d78fc044/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-15T03:13:34,361 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/c769a833-6561-0e53-79a7-5504d78fc044/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-15T03:13:34,361 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/c769a833-6561-0e53-79a7-5504d78fc044/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-15T03:13:34,362 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/c769a833-6561-0e53-79a7-5504d78fc044/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-15T03:13:34,362 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/c769a833-6561-0e53-79a7-5504d78fc044/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-15T03:13:34,362 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/c769a833-6561-0e53-79a7-5504d78fc044/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-15T03:13:34,363 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/c769a833-6561-0e53-79a7-5504d78fc044/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-15T03:13:34,363 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/c769a833-6561-0e53-79a7-5504d78fc044/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-15T03:13:34,363 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/c769a833-6561-0e53-79a7-5504d78fc044/nfs.dump.dir in system properties and HBase conf 2024-11-15T03:13:34,364 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/c769a833-6561-0e53-79a7-5504d78fc044/java.io.tmpdir in system properties and HBase conf 2024-11-15T03:13:34,364 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/c769a833-6561-0e53-79a7-5504d78fc044/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-15T03:13:34,365 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/c769a833-6561-0e53-79a7-5504d78fc044/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-15T03:13:34,365 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/c769a833-6561-0e53-79a7-5504d78fc044/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-15T03:13:35,724 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-15T03:13:35,799 INFO [Time-limited test {}] log.Log(170): Logging initialized @2775ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-15T03:13:35,869 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-15T03:13:35,928 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-15T03:13:35,949 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-15T03:13:35,950 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-15T03:13:35,951 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-15T03:13:35,963 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-15T03:13:35,966 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3912242{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/c769a833-6561-0e53-79a7-5504d78fc044/hadoop.log.dir/,AVAILABLE} 2024-11-15T03:13:35,967 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@595320c8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-15T03:13:36,157 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@64418798{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/c769a833-6561-0e53-79a7-5504d78fc044/java.io.tmpdir/jetty-localhost-45975-hadoop-hdfs-3_4_1-tests_jar-_-any-12813265166487913277/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-15T03:13:36,170 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@11744524{HTTP/1.1, (http/1.1)}{localhost:45975} 2024-11-15T03:13:36,171 INFO [Time-limited test {}] server.Server(415): Started @3147ms 2024-11-15T03:13:36,741 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-15T03:13:36,750 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-15T03:13:36,752 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-15T03:13:36,752 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-15T03:13:36,752 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-15T03:13:36,753 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@51c39823{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/c769a833-6561-0e53-79a7-5504d78fc044/hadoop.log.dir/,AVAILABLE} 2024-11-15T03:13:36,754 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@c7135d9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-15T03:13:36,857 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@65e17246{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/c769a833-6561-0e53-79a7-5504d78fc044/java.io.tmpdir/jetty-localhost-42673-hadoop-hdfs-3_4_1-tests_jar-_-any-13821515085067278993/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-15T03:13:36,858 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@220c8d80{HTTP/1.1, (http/1.1)}{localhost:42673} 2024-11-15T03:13:36,858 INFO [Time-limited test {}] server.Server(415): Started @3835ms 2024-11-15T03:13:36,907 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-15T03:13:37,026 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-15T03:13:37,033 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-15T03:13:37,034 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-15T03:13:37,034 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-15T03:13:37,035 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-15T03:13:37,036 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2ff36c4c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/c769a833-6561-0e53-79a7-5504d78fc044/hadoop.log.dir/,AVAILABLE} 2024-11-15T03:13:37,037 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6331ed3f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-15T03:13:37,148 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@395ff33a{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/c769a833-6561-0e53-79a7-5504d78fc044/java.io.tmpdir/jetty-localhost-36881-hadoop-hdfs-3_4_1-tests_jar-_-any-13994083185807718171/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-15T03:13:37,149 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4d24ac65{HTTP/1.1, (http/1.1)}{localhost:36881} 2024-11-15T03:13:37,149 INFO [Time-limited test {}] server.Server(415): Started @4126ms 2024-11-15T03:13:37,152 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-15T03:13:37,193 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-15T03:13:37,197 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-15T03:13:37,199 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-15T03:13:37,199 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-15T03:13:37,199 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-15T03:13:37,200 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6ed43cd1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/c769a833-6561-0e53-79a7-5504d78fc044/hadoop.log.dir/,AVAILABLE} 2024-11-15T03:13:37,200 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@48a2d1c6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-15T03:13:37,298 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@46041a9d{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/c769a833-6561-0e53-79a7-5504d78fc044/java.io.tmpdir/jetty-localhost-40597-hadoop-hdfs-3_4_1-tests_jar-_-any-5439963522780689733/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-15T03:13:37,298 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2a39ea25{HTTP/1.1, (http/1.1)}{localhost:40597} 2024-11-15T03:13:37,298 INFO [Time-limited test {}] server.Server(415): Started @4275ms 2024-11-15T03:13:37,300 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-15T03:13:38,576 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/c769a833-6561-0e53-79a7-5504d78fc044/cluster_03df53d9-c732-dbe9-3abf-bcf77ad8c819/data/data2/current/BP-34329870-172.17.0.2-1731640414946/current, will proceed with Du for space computation calculation, 2024-11-15T03:13:38,576 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/c769a833-6561-0e53-79a7-5504d78fc044/cluster_03df53d9-c732-dbe9-3abf-bcf77ad8c819/data/data3/current/BP-34329870-172.17.0.2-1731640414946/current, will proceed with Du for space computation calculation, 2024-11-15T03:13:38,576 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/c769a833-6561-0e53-79a7-5504d78fc044/cluster_03df53d9-c732-dbe9-3abf-bcf77ad8c819/data/data1/current/BP-34329870-172.17.0.2-1731640414946/current, will proceed with Du for space computation calculation, 2024-11-15T03:13:38,576 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/c769a833-6561-0e53-79a7-5504d78fc044/cluster_03df53d9-c732-dbe9-3abf-bcf77ad8c819/data/data4/current/BP-34329870-172.17.0.2-1731640414946/current, will proceed with Du for space computation calculation, 2024-11-15T03:13:38,619 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-15T03:13:38,619 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-15T03:13:38,638 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/c769a833-6561-0e53-79a7-5504d78fc044/cluster_03df53d9-c732-dbe9-3abf-bcf77ad8c819/data/data5/current/BP-34329870-172.17.0.2-1731640414946/current, will proceed with Du for space computation calculation, 2024-11-15T03:13:38,639 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/c769a833-6561-0e53-79a7-5504d78fc044/cluster_03df53d9-c732-dbe9-3abf-bcf77ad8c819/data/data6/current/BP-34329870-172.17.0.2-1731640414946/current, will proceed with Du for space computation calculation, 2024-11-15T03:13:38,664 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-15T03:13:38,671 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x319ad24cf529526d with lease ID 0x363a4c4abb29758b: Processing first storage report for DS-0b930937-9ceb-4d09-87e5-60bb062fe96b from datanode DatanodeRegistration(127.0.0.1:46785, datanodeUuid=2a4a5591-9f2e-402d-a66a-07984ac70fd7, infoPort=36035, infoSecurePort=0, ipcPort=43283, storageInfo=lv=-57;cid=testClusterID;nsid=1258427378;c=1731640414946) 2024-11-15T03:13:38,672 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x319ad24cf529526d with lease ID 0x363a4c4abb29758b: from storage DS-0b930937-9ceb-4d09-87e5-60bb062fe96b node DatanodeRegistration(127.0.0.1:46785, datanodeUuid=2a4a5591-9f2e-402d-a66a-07984ac70fd7, infoPort=36035, infoSecurePort=0, ipcPort=43283, storageInfo=lv=-57;cid=testClusterID;nsid=1258427378;c=1731640414946), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-15T03:13:38,672 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x550a964c018a922c with lease ID 0x363a4c4abb29758a: Processing first storage report for DS-818fe711-350c-400c-9163-61e8ae4f0904 from datanode DatanodeRegistration(127.0.0.1:36559, datanodeUuid=75738713-3387-41da-908d-ed2604ddd397, infoPort=45153, infoSecurePort=0, ipcPort=41905, storageInfo=lv=-57;cid=testClusterID;nsid=1258427378;c=1731640414946) 2024-11-15T03:13:38,673 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x550a964c018a922c with lease ID 0x363a4c4abb29758a: from storage DS-818fe711-350c-400c-9163-61e8ae4f0904 node DatanodeRegistration(127.0.0.1:36559, datanodeUuid=75738713-3387-41da-908d-ed2604ddd397, infoPort=45153, infoSecurePort=0, ipcPort=41905, storageInfo=lv=-57;cid=testClusterID;nsid=1258427378;c=1731640414946), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-15T03:13:38,673 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4535ec665c818ee8 with lease ID 0x363a4c4abb29758c: Processing first storage report for DS-e648626c-dc65-44f2-ab4f-9b6acd52971a from datanode DatanodeRegistration(127.0.0.1:38319, datanodeUuid=2937c750-c298-4ba9-8b5d-5bf6c91cdfcc, infoPort=37901, infoSecurePort=0, ipcPort=36329, storageInfo=lv=-57;cid=testClusterID;nsid=1258427378;c=1731640414946) 2024-11-15T03:13:38,673 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4535ec665c818ee8 with lease ID 0x363a4c4abb29758c: from storage DS-e648626c-dc65-44f2-ab4f-9b6acd52971a node DatanodeRegistration(127.0.0.1:38319, datanodeUuid=2937c750-c298-4ba9-8b5d-5bf6c91cdfcc, infoPort=37901, infoSecurePort=0, ipcPort=36329, storageInfo=lv=-57;cid=testClusterID;nsid=1258427378;c=1731640414946), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-15T03:13:38,673 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x319ad24cf529526d with lease ID 0x363a4c4abb29758b: Processing first storage report for DS-f651347c-93e4-4bd8-985e-ef2ab7da3cfb from datanode DatanodeRegistration(127.0.0.1:46785, datanodeUuid=2a4a5591-9f2e-402d-a66a-07984ac70fd7, infoPort=36035, infoSecurePort=0, ipcPort=43283, storageInfo=lv=-57;cid=testClusterID;nsid=1258427378;c=1731640414946) 2024-11-15T03:13:38,673 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x319ad24cf529526d with lease ID 0x363a4c4abb29758b: from storage DS-f651347c-93e4-4bd8-985e-ef2ab7da3cfb node DatanodeRegistration(127.0.0.1:46785, datanodeUuid=2a4a5591-9f2e-402d-a66a-07984ac70fd7, infoPort=36035, infoSecurePort=0, ipcPort=43283, storageInfo=lv=-57;cid=testClusterID;nsid=1258427378;c=1731640414946), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-15T03:13:38,674 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x550a964c018a922c with lease ID 0x363a4c4abb29758a: Processing first storage report for DS-933e738b-f8d1-40dc-b8eb-1fa3127d335f from datanode DatanodeRegistration(127.0.0.1:36559, datanodeUuid=75738713-3387-41da-908d-ed2604ddd397, infoPort=45153, infoSecurePort=0, ipcPort=41905, storageInfo=lv=-57;cid=testClusterID;nsid=1258427378;c=1731640414946) 2024-11-15T03:13:38,674 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x550a964c018a922c with lease ID 0x363a4c4abb29758a: from storage DS-933e738b-f8d1-40dc-b8eb-1fa3127d335f node DatanodeRegistration(127.0.0.1:36559, datanodeUuid=75738713-3387-41da-908d-ed2604ddd397, infoPort=45153, infoSecurePort=0, ipcPort=41905, storageInfo=lv=-57;cid=testClusterID;nsid=1258427378;c=1731640414946), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-15T03:13:38,674 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4535ec665c818ee8 with lease ID 0x363a4c4abb29758c: Processing first storage report for DS-416d3d6d-619e-4d03-9270-1015b9156e56 from datanode DatanodeRegistration(127.0.0.1:38319, datanodeUuid=2937c750-c298-4ba9-8b5d-5bf6c91cdfcc, infoPort=37901, infoSecurePort=0, ipcPort=36329, storageInfo=lv=-57;cid=testClusterID;nsid=1258427378;c=1731640414946) 2024-11-15T03:13:38,674 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4535ec665c818ee8 with lease ID 0x363a4c4abb29758c: from storage DS-416d3d6d-619e-4d03-9270-1015b9156e56 node DatanodeRegistration(127.0.0.1:38319, datanodeUuid=2937c750-c298-4ba9-8b5d-5bf6c91cdfcc, infoPort=37901, infoSecurePort=0, ipcPort=36329, storageInfo=lv=-57;cid=testClusterID;nsid=1258427378;c=1731640414946), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-15T03:13:38,730 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/c769a833-6561-0e53-79a7-5504d78fc044 2024-11-15T03:13:38,790 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/c769a833-6561-0e53-79a7-5504d78fc044/cluster_03df53d9-c732-dbe9-3abf-bcf77ad8c819/zookeeper_0, clientPort=64070, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/c769a833-6561-0e53-79a7-5504d78fc044/cluster_03df53d9-c732-dbe9-3abf-bcf77ad8c819/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/c769a833-6561-0e53-79a7-5504d78fc044/cluster_03df53d9-c732-dbe9-3abf-bcf77ad8c819/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-15T03:13:38,798 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=64070 2024-11-15T03:13:38,810 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T03:13:38,812 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T03:13:39,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46785 is added to blk_1073741825_1001 (size=7) 2024-11-15T03:13:39,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36559 is added to blk_1073741825_1001 (size=7) 2024-11-15T03:13:39,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38319 is added to blk_1073741825_1001 (size=7) 2024-11-15T03:13:39,427 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46 with version=8 2024-11-15T03:13:39,428 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/hbase-staging 2024-11-15T03:13:39,498 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-15T03:13:39,734 INFO [Time-limited test {}] client.ConnectionUtils(128): master/91f82191efb8:0 server-side Connection retries=45 2024-11-15T03:13:39,742 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T03:13:39,742 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-15T03:13:39,747 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-15T03:13:39,747 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T03:13:39,747 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-15T03:13:39,870 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-15T03:13:39,921 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-15T03:13:39,930 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-15T03:13:39,933 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T03:13:39,956 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 3737 (auto-detected) 2024-11-15T03:13:39,957 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-15T03:13:39,974 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42051 2024-11-15T03:13:39,994 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42051 connecting to ZooKeeper ensemble=127.0.0.1:64070 2024-11-15T03:13:40,121 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:420510x0, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-15T03:13:40,125 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42051-0x1013c7bdd830000 connected 2024-11-15T03:13:40,233 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T03:13:40,237 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T03:13:40,251 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42051-0x1013c7bdd830000, quorum=127.0.0.1:64070, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T03:13:40,257 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46, hbase.cluster.distributed=false 2024-11-15T03:13:40,287 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42051-0x1013c7bdd830000, quorum=127.0.0.1:64070, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-15T03:13:40,293 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42051 2024-11-15T03:13:40,296 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42051 2024-11-15T03:13:40,300 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42051 2024-11-15T03:13:40,303 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42051 2024-11-15T03:13:40,304 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42051 2024-11-15T03:13:40,421 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/91f82191efb8:0 server-side Connection retries=45 2024-11-15T03:13:40,422 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T03:13:40,422 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-15T03:13:40,423 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-15T03:13:40,423 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T03:13:40,423 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-15T03:13:40,426 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-15T03:13:40,429 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T03:13:40,430 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42027 2024-11-15T03:13:40,432 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42027 connecting to ZooKeeper ensemble=127.0.0.1:64070 2024-11-15T03:13:40,433 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T03:13:40,438 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T03:13:40,552 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:420270x0, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-15T03:13:40,553 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:420270x0, quorum=127.0.0.1:64070, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T03:13:40,553 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42027-0x1013c7bdd830001 connected 2024-11-15T03:13:40,558 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-15T03:13:40,567 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-15T03:13:40,570 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42027-0x1013c7bdd830001, quorum=127.0.0.1:64070, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-15T03:13:40,576 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42027-0x1013c7bdd830001, quorum=127.0.0.1:64070, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-15T03:13:40,577 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42027 2024-11-15T03:13:40,577 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42027 2024-11-15T03:13:40,577 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42027 2024-11-15T03:13:40,578 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42027 2024-11-15T03:13:40,578 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42027 2024-11-15T03:13:40,593 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/91f82191efb8:0 server-side Connection retries=45 2024-11-15T03:13:40,594 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T03:13:40,594 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-15T03:13:40,594 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-15T03:13:40,594 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T03:13:40,594 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-15T03:13:40,595 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-15T03:13:40,595 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T03:13:40,596 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37523 2024-11-15T03:13:40,597 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37523 connecting to ZooKeeper ensemble=127.0.0.1:64070 2024-11-15T03:13:40,598 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T03:13:40,601 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T03:13:40,615 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:375230x0, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-15T03:13:40,616 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37523-0x1013c7bdd830002, quorum=127.0.0.1:64070, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T03:13:40,616 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37523-0x1013c7bdd830002 connected 2024-11-15T03:13:40,616 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-15T03:13:40,618 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-15T03:13:40,619 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37523-0x1013c7bdd830002, quorum=127.0.0.1:64070, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-15T03:13:40,620 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37523-0x1013c7bdd830002, quorum=127.0.0.1:64070, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-15T03:13:40,621 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37523 2024-11-15T03:13:40,621 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37523 2024-11-15T03:13:40,622 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37523 2024-11-15T03:13:40,622 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37523 2024-11-15T03:13:40,623 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37523 2024-11-15T03:13:40,638 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/91f82191efb8:0 server-side Connection retries=45 2024-11-15T03:13:40,639 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T03:13:40,639 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-15T03:13:40,639 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-15T03:13:40,639 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T03:13:40,639 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-15T03:13:40,639 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-15T03:13:40,639 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T03:13:40,640 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41451 2024-11-15T03:13:40,642 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41451 connecting to ZooKeeper ensemble=127.0.0.1:64070 2024-11-15T03:13:40,643 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T03:13:40,645 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T03:13:40,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:414510x0, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-15T03:13:40,657 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:414510x0, quorum=127.0.0.1:64070, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T03:13:40,658 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41451-0x1013c7bdd830003 connected 2024-11-15T03:13:40,658 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-15T03:13:40,659 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-15T03:13:40,660 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41451-0x1013c7bdd830003, quorum=127.0.0.1:64070, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-15T03:13:40,661 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41451-0x1013c7bdd830003, quorum=127.0.0.1:64070, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-15T03:13:40,662 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41451 2024-11-15T03:13:40,662 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41451 2024-11-15T03:13:40,663 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41451 2024-11-15T03:13:40,663 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41451 2024-11-15T03:13:40,664 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41451 2024-11-15T03:13:40,681 DEBUG [M:0;91f82191efb8:42051 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;91f82191efb8:42051 2024-11-15T03:13:40,682 INFO [master/91f82191efb8:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/91f82191efb8,42051,1731640419585 2024-11-15T03:13:40,699 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42027-0x1013c7bdd830001, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T03:13:40,699 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41451-0x1013c7bdd830003, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T03:13:40,699 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37523-0x1013c7bdd830002, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T03:13:40,699 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42051-0x1013c7bdd830000, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T03:13:40,701 DEBUG [master/91f82191efb8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42051-0x1013c7bdd830000, quorum=127.0.0.1:64070, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/91f82191efb8,42051,1731640419585 2024-11-15T03:13:40,730 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37523-0x1013c7bdd830002, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-15T03:13:40,730 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42027-0x1013c7bdd830001, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-15T03:13:40,730 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41451-0x1013c7bdd830003, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-15T03:13:40,730 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42051-0x1013c7bdd830000, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T03:13:40,731 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37523-0x1013c7bdd830002, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T03:13:40,731 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41451-0x1013c7bdd830003, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T03:13:40,731 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42027-0x1013c7bdd830001, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T03:13:40,732 DEBUG [master/91f82191efb8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42051-0x1013c7bdd830000, quorum=127.0.0.1:64070, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-15T03:13:40,733 INFO [master/91f82191efb8:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/91f82191efb8,42051,1731640419585 from backup master directory 2024-11-15T03:13:40,747 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37523-0x1013c7bdd830002, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T03:13:40,747 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42051-0x1013c7bdd830000, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/91f82191efb8,42051,1731640419585 2024-11-15T03:13:40,747 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42027-0x1013c7bdd830001, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T03:13:40,747 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42051-0x1013c7bdd830000, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T03:13:40,747 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41451-0x1013c7bdd830003, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T03:13:40,748 WARN [master/91f82191efb8:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-15T03:13:40,749 INFO [master/91f82191efb8:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=91f82191efb8,42051,1731640419585 2024-11-15T03:13:40,751 INFO [master/91f82191efb8:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-15T03:13:40,753 INFO [master/91f82191efb8:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-15T03:13:40,808 DEBUG [master/91f82191efb8:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/hbase.id] with ID: 451b97df-43ba-4345-aef9-5c28b9dc1656 2024-11-15T03:13:40,808 DEBUG [master/91f82191efb8:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/.tmp/hbase.id 2024-11-15T03:13:40,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38319 is added to blk_1073741826_1002 (size=42) 2024-11-15T03:13:40,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46785 is added to blk_1073741826_1002 (size=42) 2024-11-15T03:13:40,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36559 is added to blk_1073741826_1002 (size=42) 2024-11-15T03:13:40,822 DEBUG [master/91f82191efb8:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/.tmp/hbase.id]:[hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/hbase.id] 2024-11-15T03:13:40,864 INFO [master/91f82191efb8:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T03:13:40,869 INFO [master/91f82191efb8:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-15T03:13:40,888 INFO [master/91f82191efb8:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-15T03:13:40,920 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37523-0x1013c7bdd830002, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T03:13:40,920 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42051-0x1013c7bdd830000, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T03:13:40,920 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42027-0x1013c7bdd830001, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T03:13:40,920 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41451-0x1013c7bdd830003, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T03:13:40,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36559 is added to blk_1073741827_1003 (size=196) 2024-11-15T03:13:40,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38319 is added to blk_1073741827_1003 (size=196) 2024-11-15T03:13:40,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46785 is added to blk_1073741827_1003 (size=196) 2024-11-15T03:13:40,962 INFO [master/91f82191efb8:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T03:13:40,964 INFO [master/91f82191efb8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-15T03:13:40,978 DEBUG [master/91f82191efb8:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T03:13:40,981 INFO [master/91f82191efb8:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T03:13:41,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46785 is added to blk_1073741828_1004 (size=1189) 2024-11-15T03:13:41,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38319 is added to blk_1073741828_1004 (size=1189) 2024-11-15T03:13:41,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36559 is added to blk_1073741828_1004 (size=1189) 2024-11-15T03:13:41,030 INFO [master/91f82191efb8:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/MasterData/data/master/store 2024-11-15T03:13:41,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46785 is added to blk_1073741829_1005 (size=34) 2024-11-15T03:13:41,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38319 is added to blk_1073741829_1005 (size=34) 2024-11-15T03:13:41,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36559 is added to blk_1073741829_1005 (size=34) 2024-11-15T03:13:41,057 INFO [master/91f82191efb8:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-15T03:13:41,060 DEBUG [master/91f82191efb8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T03:13:41,061 DEBUG [master/91f82191efb8:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-15T03:13:41,062 INFO [master/91f82191efb8:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T03:13:41,062 DEBUG [master/91f82191efb8:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T03:13:41,063 DEBUG [master/91f82191efb8:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-15T03:13:41,064 DEBUG [master/91f82191efb8:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T03:13:41,064 INFO [master/91f82191efb8:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T03:13:41,065 DEBUG [master/91f82191efb8:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731640421061Disabling compacts and flushes for region at 1731640421061Disabling writes for close at 1731640421064 (+3 ms)Writing region close event to WAL at 1731640421064Closed at 1731640421064 2024-11-15T03:13:41,067 WARN [master/91f82191efb8:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/MasterData/data/master/store/.initializing 2024-11-15T03:13:41,067 DEBUG [master/91f82191efb8:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/MasterData/WALs/91f82191efb8,42051,1731640419585 2024-11-15T03:13:41,075 INFO [master/91f82191efb8:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-15T03:13:41,088 INFO [master/91f82191efb8:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=91f82191efb8%2C42051%2C1731640419585, suffix=, logDir=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/MasterData/WALs/91f82191efb8,42051,1731640419585, archiveDir=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/MasterData/oldWALs, maxLogs=10 2024-11-15T03:13:41,119 DEBUG [master/91f82191efb8:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/MasterData/WALs/91f82191efb8,42051,1731640419585/91f82191efb8%2C42051%2C1731640419585.1731640421094, exclude list is [], retry=0 2024-11-15T03:13:41,136 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36559,DS-818fe711-350c-400c-9163-61e8ae4f0904,DISK] 2024-11-15T03:13:41,136 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38319,DS-e648626c-dc65-44f2-ab4f-9b6acd52971a,DISK] 2024-11-15T03:13:41,136 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46785,DS-0b930937-9ceb-4d09-87e5-60bb062fe96b,DISK] 2024-11-15T03:13:41,140 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-15T03:13:41,174 INFO [master/91f82191efb8:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/MasterData/WALs/91f82191efb8,42051,1731640419585/91f82191efb8%2C42051%2C1731640419585.1731640421094 2024-11-15T03:13:41,175 DEBUG [master/91f82191efb8:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37901:37901),(127.0.0.1/127.0.0.1:45153:45153),(127.0.0.1/127.0.0.1:36035:36035)] 2024-11-15T03:13:41,175 DEBUG [master/91f82191efb8:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-15T03:13:41,176 DEBUG [master/91f82191efb8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T03:13:41,179 DEBUG [master/91f82191efb8:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T03:13:41,180 DEBUG [master/91f82191efb8:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T03:13:41,217 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T03:13:41,242 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-15T03:13:41,245 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T03:13:41,248 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T03:13:41,249 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T03:13:41,252 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-15T03:13:41,252 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T03:13:41,253 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T03:13:41,253 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T03:13:41,256 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-15T03:13:41,256 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T03:13:41,258 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T03:13:41,258 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T03:13:41,261 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-15T03:13:41,261 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T03:13:41,262 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T03:13:41,263 DEBUG [master/91f82191efb8:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T03:13:41,266 DEBUG [master/91f82191efb8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-15T03:13:41,267 DEBUG [master/91f82191efb8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-15T03:13:41,272 DEBUG [master/91f82191efb8:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T03:13:41,273 DEBUG [master/91f82191efb8:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T03:13:41,276 DEBUG [master/91f82191efb8:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-15T03:13:41,280 DEBUG [master/91f82191efb8:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T03:13:41,285 DEBUG [master/91f82191efb8:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T03:13:41,286 INFO [master/91f82191efb8:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65001050, jitterRate=-0.031408876180648804}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-15T03:13:41,293 DEBUG [master/91f82191efb8:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731640421192Initializing all the Stores at 1731640421194 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731640421195 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731640421195Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731640421196 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731640421196Cleaning up temporary data from old regions at 1731640421273 (+77 ms)Region opened successfully at 1731640421293 (+20 ms) 2024-11-15T03:13:41,294 INFO [master/91f82191efb8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-15T03:13:41,329 DEBUG [master/91f82191efb8:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28ead083, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=91f82191efb8/172.17.0.2:0 2024-11-15T03:13:41,362 INFO [master/91f82191efb8:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-15T03:13:41,372 INFO [master/91f82191efb8:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T03:13:41,372 INFO [master/91f82191efb8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T03:13:41,374 INFO [master/91f82191efb8:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T03:13:41,376 INFO [master/91f82191efb8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-15T03:13:41,381 INFO [master/91f82191efb8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-15T03:13:41,381 INFO [master/91f82191efb8:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-15T03:13:41,407 INFO [master/91f82191efb8:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-15T03:13:41,415 DEBUG [master/91f82191efb8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42051-0x1013c7bdd830000, quorum=127.0.0.1:64070, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-15T03:13:41,473 DEBUG [master/91f82191efb8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-15T03:13:41,477 INFO [master/91f82191efb8:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-15T03:13:41,479 DEBUG [master/91f82191efb8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42051-0x1013c7bdd830000, quorum=127.0.0.1:64070, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-15T03:13:41,488 DEBUG [master/91f82191efb8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-15T03:13:41,491 INFO [master/91f82191efb8:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-15T03:13:41,497 DEBUG [master/91f82191efb8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42051-0x1013c7bdd830000, quorum=127.0.0.1:64070, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-15T03:13:41,509 DEBUG [master/91f82191efb8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-15T03:13:41,511 DEBUG [master/91f82191efb8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42051-0x1013c7bdd830000, quorum=127.0.0.1:64070, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-15T03:13:41,520 DEBUG [master/91f82191efb8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-15T03:13:41,537 DEBUG [master/91f82191efb8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42051-0x1013c7bdd830000, quorum=127.0.0.1:64070, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-15T03:13:41,546 DEBUG [master/91f82191efb8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-15T03:13:41,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37523-0x1013c7bdd830002, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-15T03:13:41,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41451-0x1013c7bdd830003, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-15T03:13:41,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42051-0x1013c7bdd830000, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-15T03:13:41,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37523-0x1013c7bdd830002, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T03:13:41,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41451-0x1013c7bdd830003, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T03:13:41,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42051-0x1013c7bdd830000, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T03:13:41,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42027-0x1013c7bdd830001, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-15T03:13:41,558 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42027-0x1013c7bdd830001, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T03:13:41,561 INFO [master/91f82191efb8:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=91f82191efb8,42051,1731640419585, sessionid=0x1013c7bdd830000, setting cluster-up flag (Was=false) 2024-11-15T03:13:41,594 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37523-0x1013c7bdd830002, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T03:13:41,594 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42027-0x1013c7bdd830001, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T03:13:41,594 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42051-0x1013c7bdd830000, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T03:13:41,594 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41451-0x1013c7bdd830003, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T03:13:41,626 DEBUG [master/91f82191efb8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-15T03:13:41,629 DEBUG [master/91f82191efb8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=91f82191efb8,42051,1731640419585 2024-11-15T03:13:41,652 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37523-0x1013c7bdd830002, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T03:13:41,652 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42027-0x1013c7bdd830001, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T03:13:41,652 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42051-0x1013c7bdd830000, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T03:13:41,652 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41451-0x1013c7bdd830003, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T03:13:41,683 DEBUG [master/91f82191efb8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-15T03:13:41,685 DEBUG [master/91f82191efb8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=91f82191efb8,42051,1731640419585 2024-11-15T03:13:41,694 INFO [master/91f82191efb8:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-15T03:13:41,763 DEBUG [master/91f82191efb8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T03:13:41,767 INFO [RS:2;91f82191efb8:41451 {}] regionserver.HRegionServer(746): ClusterId : 451b97df-43ba-4345-aef9-5c28b9dc1656 2024-11-15T03:13:41,767 INFO [RS:0;91f82191efb8:42027 {}] regionserver.HRegionServer(746): ClusterId : 451b97df-43ba-4345-aef9-5c28b9dc1656 2024-11-15T03:13:41,767 INFO [RS:1;91f82191efb8:37523 {}] regionserver.HRegionServer(746): ClusterId : 451b97df-43ba-4345-aef9-5c28b9dc1656 2024-11-15T03:13:41,770 DEBUG [RS:1;91f82191efb8:37523 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-15T03:13:41,770 DEBUG [RS:2;91f82191efb8:41451 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-15T03:13:41,770 DEBUG [RS:0;91f82191efb8:42027 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-15T03:13:41,773 INFO [master/91f82191efb8:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-15T03:13:41,779 INFO [master/91f82191efb8:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-15T03:13:41,785 DEBUG [RS:2;91f82191efb8:41451 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-15T03:13:41,785 DEBUG [RS:1;91f82191efb8:37523 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-15T03:13:41,785 DEBUG [RS:0;91f82191efb8:42027 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-15T03:13:41,785 DEBUG [RS:2;91f82191efb8:41451 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-15T03:13:41,785 DEBUG [RS:1;91f82191efb8:37523 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-15T03:13:41,785 DEBUG [RS:0;91f82191efb8:42027 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-15T03:13:41,785 DEBUG [master/91f82191efb8:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 91f82191efb8,42051,1731640419585 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-15T03:13:41,795 DEBUG [RS:0;91f82191efb8:42027 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-15T03:13:41,795 DEBUG [RS:2;91f82191efb8:41451 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-15T03:13:41,795 DEBUG [RS:1;91f82191efb8:37523 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-15T03:13:41,795 DEBUG [RS:0;91f82191efb8:42027 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5610cd88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=91f82191efb8/172.17.0.2:0 2024-11-15T03:13:41,795 DEBUG [RS:2;91f82191efb8:41451 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7eaeb363, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=91f82191efb8/172.17.0.2:0 2024-11-15T03:13:41,795 DEBUG [RS:1;91f82191efb8:37523 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19acbf44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=91f82191efb8/172.17.0.2:0 2024-11-15T03:13:41,796 DEBUG [master/91f82191efb8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/91f82191efb8:0, corePoolSize=5, maxPoolSize=5 2024-11-15T03:13:41,796 DEBUG [master/91f82191efb8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/91f82191efb8:0, corePoolSize=5, maxPoolSize=5 2024-11-15T03:13:41,797 DEBUG [master/91f82191efb8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/91f82191efb8:0, corePoolSize=5, maxPoolSize=5 2024-11-15T03:13:41,797 DEBUG [master/91f82191efb8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/91f82191efb8:0, corePoolSize=5, maxPoolSize=5 2024-11-15T03:13:41,797 DEBUG [master/91f82191efb8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/91f82191efb8:0, corePoolSize=10, maxPoolSize=10 2024-11-15T03:13:41,797 DEBUG [master/91f82191efb8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/91f82191efb8:0, corePoolSize=1, maxPoolSize=1 2024-11-15T03:13:41,797 DEBUG [master/91f82191efb8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/91f82191efb8:0, corePoolSize=2, maxPoolSize=2 2024-11-15T03:13:41,798 DEBUG [master/91f82191efb8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/91f82191efb8:0, corePoolSize=1, maxPoolSize=1 2024-11-15T03:13:41,805 INFO [master/91f82191efb8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731640451805 2024-11-15T03:13:41,807 INFO [master/91f82191efb8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-15T03:13:41,809 INFO [master/91f82191efb8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-15T03:13:41,811 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-15T03:13:41,812 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-15T03:13:41,813 INFO [master/91f82191efb8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-15T03:13:41,813 DEBUG [RS:0;91f82191efb8:42027 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;91f82191efb8:42027 2024-11-15T03:13:41,813 INFO [master/91f82191efb8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-15T03:13:41,814 INFO [master/91f82191efb8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-15T03:13:41,814 INFO [master/91f82191efb8:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-15T03:13:41,815 DEBUG [RS:1;91f82191efb8:37523 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;91f82191efb8:37523 2024-11-15T03:13:41,815 INFO [master/91f82191efb8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-15T03:13:41,817 DEBUG [RS:2;91f82191efb8:41451 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;91f82191efb8:41451 2024-11-15T03:13:41,817 INFO [RS:0;91f82191efb8:42027 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-15T03:13:41,817 INFO [RS:1;91f82191efb8:37523 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-15T03:13:41,817 INFO [RS:0;91f82191efb8:42027 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-15T03:13:41,817 INFO [RS:1;91f82191efb8:37523 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-15T03:13:41,817 INFO [RS:2;91f82191efb8:41451 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-15T03:13:41,817 DEBUG [RS:1;91f82191efb8:37523 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-15T03:13:41,817 DEBUG [RS:0;91f82191efb8:42027 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-15T03:13:41,817 INFO [RS:2;91f82191efb8:41451 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-15T03:13:41,817 DEBUG [RS:2;91f82191efb8:41451 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-15T03:13:41,819 INFO [master/91f82191efb8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-15T03:13:41,820 INFO [master/91f82191efb8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-15T03:13:41,820 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T03:13:41,820 INFO [RS:1;91f82191efb8:37523 {}] regionserver.HRegionServer(2659): reportForDuty to master=91f82191efb8,42051,1731640419585 with port=37523, startcode=1731640420593 2024-11-15T03:13:41,820 INFO [RS:0;91f82191efb8:42027 {}] regionserver.HRegionServer(2659): reportForDuty to master=91f82191efb8,42051,1731640419585 with port=42027, startcode=1731640420387 2024-11-15T03:13:41,820 INFO [RS:2;91f82191efb8:41451 {}] regionserver.HRegionServer(2659): reportForDuty to master=91f82191efb8,42051,1731640419585 with port=41451, startcode=1731640420638 2024-11-15T03:13:41,821 INFO [master/91f82191efb8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-15T03:13:41,820 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-15T03:13:41,827 INFO [master/91f82191efb8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-15T03:13:41,828 INFO [master/91f82191efb8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-15T03:13:41,832 DEBUG [RS:2;91f82191efb8:41451 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-15T03:13:41,832 DEBUG [RS:0;91f82191efb8:42027 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-15T03:13:41,832 DEBUG [RS:1;91f82191efb8:37523 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-15T03:13:41,833 DEBUG [master/91f82191efb8:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/91f82191efb8:0:becomeActiveMaster-HFileCleaner.large.0-1731640421829,5,FailOnTimeoutGroup] 2024-11-15T03:13:41,834 DEBUG [master/91f82191efb8:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/91f82191efb8:0:becomeActiveMaster-HFileCleaner.small.0-1731640421833,5,FailOnTimeoutGroup] 2024-11-15T03:13:41,834 INFO [master/91f82191efb8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-15T03:13:41,834 INFO [master/91f82191efb8:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-15T03:13:41,835 INFO [master/91f82191efb8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-15T03:13:41,835 INFO [master/91f82191efb8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-15T03:13:41,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38319 is added to blk_1073741831_1007 (size=1321) 2024-11-15T03:13:41,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36559 is added to blk_1073741831_1007 (size=1321) 2024-11-15T03:13:41,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46785 is added to blk_1073741831_1007 (size=1321) 2024-11-15T03:13:41,848 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-15T03:13:41,849 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46 2024-11-15T03:13:41,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36559 is added to blk_1073741832_1008 (size=32) 2024-11-15T03:13:41,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46785 is added to blk_1073741832_1008 (size=32) 2024-11-15T03:13:41,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38319 is added to blk_1073741832_1008 (size=32) 2024-11-15T03:13:41,880 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T03:13:41,883 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-15T03:13:41,885 INFO [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58933, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-15T03:13:41,885 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46505, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-15T03:13:41,885 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44087, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-15T03:13:41,887 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-15T03:13:41,887 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T03:13:41,889 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T03:13:41,889 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-15T03:13:41,892 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-15T03:13:41,892 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T03:13:41,893 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42051 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 91f82191efb8,42027,1731640420387 2024-11-15T03:13:41,893 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T03:13:41,893 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-15T03:13:41,896 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-15T03:13:41,896 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T03:13:41,896 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42051 {}] master.ServerManager(517): Registering regionserver=91f82191efb8,42027,1731640420387 2024-11-15T03:13:41,897 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T03:13:41,897 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-15T03:13:41,900 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-15T03:13:41,901 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T03:13:41,902 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T03:13:41,902 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-15T03:13:41,904 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/hbase/meta/1588230740 2024-11-15T03:13:41,905 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/hbase/meta/1588230740 2024-11-15T03:13:41,908 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-15T03:13:41,908 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-15T03:13:41,909 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42051 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 91f82191efb8,37523,1731640420593 2024-11-15T03:13:41,909 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42051 {}] master.ServerManager(517): Registering regionserver=91f82191efb8,37523,1731640420593 2024-11-15T03:13:41,909 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-15T03:13:41,913 DEBUG [RS:0;91f82191efb8:42027 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46 2024-11-15T03:13:41,914 DEBUG [RS:0;91f82191efb8:42027 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38383 2024-11-15T03:13:41,914 DEBUG [RS:0;91f82191efb8:42027 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-15T03:13:41,916 DEBUG [RS:1;91f82191efb8:37523 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46 2024-11-15T03:13:41,916 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42051 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 91f82191efb8,41451,1731640420638 2024-11-15T03:13:41,916 DEBUG [RS:1;91f82191efb8:37523 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38383 2024-11-15T03:13:41,916 DEBUG [RS:1;91f82191efb8:37523 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-15T03:13:41,916 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42051 {}] master.ServerManager(517): Registering regionserver=91f82191efb8,41451,1731640420638 2024-11-15T03:13:41,917 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-15T03:13:41,920 DEBUG [RS:2;91f82191efb8:41451 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46 2024-11-15T03:13:41,921 DEBUG [RS:2;91f82191efb8:41451 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38383 2024-11-15T03:13:41,921 DEBUG [RS:2;91f82191efb8:41451 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-15T03:13:41,922 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T03:13:41,924 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66514738, jitterRate=-0.008853167295455933}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-15T03:13:41,929 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731640421880Initializing all the Stores at 1731640421882 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731640421882Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731640421883 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731640421883Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731640421883Cleaning up temporary data from old regions at 1731640421908 (+25 ms)Region opened successfully at 1731640421929 (+21 ms) 2024-11-15T03:13:41,930 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-15T03:13:41,930 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-15T03:13:41,930 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-15T03:13:41,930 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-15T03:13:41,930 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-15T03:13:41,930 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42051-0x1013c7bdd830000, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-15T03:13:41,932 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-15T03:13:41,932 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731640421930Disabling compacts and flushes for region at 1731640421930Disabling writes for close at 1731640421930Writing region close event to WAL at 1731640421931 (+1 ms)Closed at 1731640421932 (+1 ms) 2024-11-15T03:13:41,935 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-15T03:13:41,935 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-15T03:13:41,942 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-15T03:13:41,953 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T03:13:41,956 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-15T03:13:41,970 DEBUG [RS:0;91f82191efb8:42027 {}] zookeeper.ZKUtil(111): regionserver:42027-0x1013c7bdd830001, quorum=127.0.0.1:64070, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/91f82191efb8,42027,1731640420387 2024-11-15T03:13:41,970 WARN [RS:0;91f82191efb8:42027 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-15T03:13:41,970 DEBUG [RS:1;91f82191efb8:37523 {}] zookeeper.ZKUtil(111): regionserver:37523-0x1013c7bdd830002, quorum=127.0.0.1:64070, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/91f82191efb8,37523,1731640420593 2024-11-15T03:13:41,970 INFO [RS:0;91f82191efb8:42027 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T03:13:41,970 DEBUG [RS:2;91f82191efb8:41451 {}] zookeeper.ZKUtil(111): regionserver:41451-0x1013c7bdd830003, quorum=127.0.0.1:64070, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/91f82191efb8,41451,1731640420638 2024-11-15T03:13:41,970 WARN [RS:1;91f82191efb8:37523 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-15T03:13:41,971 WARN [RS:2;91f82191efb8:41451 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-15T03:13:41,971 INFO [RS:1;91f82191efb8:37523 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T03:13:41,971 INFO [RS:2;91f82191efb8:41451 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T03:13:41,971 DEBUG [RS:0;91f82191efb8:42027 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/WALs/91f82191efb8,42027,1731640420387 2024-11-15T03:13:41,971 DEBUG [RS:1;91f82191efb8:37523 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/WALs/91f82191efb8,37523,1731640420593 2024-11-15T03:13:41,971 DEBUG [RS:2;91f82191efb8:41451 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/WALs/91f82191efb8,41451,1731640420638 2024-11-15T03:13:41,972 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [91f82191efb8,42027,1731640420387] 2024-11-15T03:13:41,972 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [91f82191efb8,41451,1731640420638] 2024-11-15T03:13:41,972 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [91f82191efb8,37523,1731640420593] 2024-11-15T03:13:41,996 INFO [RS:1;91f82191efb8:37523 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-15T03:13:41,996 INFO [RS:0;91f82191efb8:42027 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-15T03:13:41,996 INFO [RS:2;91f82191efb8:41451 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-15T03:13:42,012 INFO [RS:0;91f82191efb8:42027 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-15T03:13:42,012 INFO [RS:1;91f82191efb8:37523 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-15T03:13:42,012 INFO [RS:2;91f82191efb8:41451 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-15T03:13:42,017 INFO [RS:2;91f82191efb8:41451 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-15T03:13:42,017 INFO [RS:1;91f82191efb8:37523 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-15T03:13:42,017 INFO [RS:0;91f82191efb8:42027 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-15T03:13:42,017 INFO [RS:2;91f82191efb8:41451 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T03:13:42,017 INFO [RS:1;91f82191efb8:37523 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T03:13:42,017 INFO [RS:0;91f82191efb8:42027 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T03:13:42,019 INFO [RS:2;91f82191efb8:41451 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-15T03:13:42,019 INFO [RS:1;91f82191efb8:37523 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-15T03:13:42,019 INFO [RS:0;91f82191efb8:42027 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-15T03:13:42,024 INFO [RS:1;91f82191efb8:37523 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-15T03:13:42,024 INFO [RS:0;91f82191efb8:42027 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-15T03:13:42,024 INFO [RS:2;91f82191efb8:41451 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-15T03:13:42,026 INFO [RS:0;91f82191efb8:42027 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-15T03:13:42,026 INFO [RS:1;91f82191efb8:37523 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-15T03:13:42,026 INFO [RS:2;91f82191efb8:41451 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-15T03:13:42,026 DEBUG [RS:0;91f82191efb8:42027 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/91f82191efb8:0, corePoolSize=1, maxPoolSize=1 2024-11-15T03:13:42,026 DEBUG [RS:1;91f82191efb8:37523 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/91f82191efb8:0, corePoolSize=1, maxPoolSize=1 2024-11-15T03:13:42,026 DEBUG [RS:2;91f82191efb8:41451 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/91f82191efb8:0, corePoolSize=1, maxPoolSize=1 2024-11-15T03:13:42,027 DEBUG [RS:1;91f82191efb8:37523 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/91f82191efb8:0, corePoolSize=1, maxPoolSize=1 2024-11-15T03:13:42,027 DEBUG [RS:0;91f82191efb8:42027 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/91f82191efb8:0, corePoolSize=1, maxPoolSize=1 2024-11-15T03:13:42,027 DEBUG [RS:2;91f82191efb8:41451 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/91f82191efb8:0, corePoolSize=1, maxPoolSize=1 2024-11-15T03:13:42,027 DEBUG [RS:1;91f82191efb8:37523 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/91f82191efb8:0, corePoolSize=1, maxPoolSize=1 2024-11-15T03:13:42,027 DEBUG [RS:0;91f82191efb8:42027 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/91f82191efb8:0, corePoolSize=1, maxPoolSize=1 2024-11-15T03:13:42,027 DEBUG [RS:2;91f82191efb8:41451 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/91f82191efb8:0, corePoolSize=1, maxPoolSize=1 2024-11-15T03:13:42,027 DEBUG [RS:1;91f82191efb8:37523 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/91f82191efb8:0, corePoolSize=1, maxPoolSize=1 2024-11-15T03:13:42,027 DEBUG [RS:0;91f82191efb8:42027 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/91f82191efb8:0, corePoolSize=1, maxPoolSize=1 2024-11-15T03:13:42,027 DEBUG [RS:1;91f82191efb8:37523 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/91f82191efb8:0, corePoolSize=1, maxPoolSize=1 2024-11-15T03:13:42,027 DEBUG [RS:2;91f82191efb8:41451 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/91f82191efb8:0, corePoolSize=1, maxPoolSize=1 2024-11-15T03:13:42,027 DEBUG [RS:1;91f82191efb8:37523 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/91f82191efb8:0, corePoolSize=2, maxPoolSize=2 2024-11-15T03:13:42,027 DEBUG [RS:0;91f82191efb8:42027 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/91f82191efb8:0, corePoolSize=1, maxPoolSize=1 2024-11-15T03:13:42,027 DEBUG [RS:2;91f82191efb8:41451 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/91f82191efb8:0, corePoolSize=1, maxPoolSize=1 2024-11-15T03:13:42,027 DEBUG [RS:1;91f82191efb8:37523 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/91f82191efb8:0, corePoolSize=1, maxPoolSize=1 2024-11-15T03:13:42,027 DEBUG [RS:0;91f82191efb8:42027 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/91f82191efb8:0, corePoolSize=2, maxPoolSize=2 2024-11-15T03:13:42,027 DEBUG [RS:1;91f82191efb8:37523 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/91f82191efb8:0, corePoolSize=1, maxPoolSize=1 2024-11-15T03:13:42,027 DEBUG [RS:2;91f82191efb8:41451 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/91f82191efb8:0, corePoolSize=2, maxPoolSize=2 2024-11-15T03:13:42,027 DEBUG [RS:0;91f82191efb8:42027 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/91f82191efb8:0, corePoolSize=1, maxPoolSize=1 2024-11-15T03:13:42,027 DEBUG [RS:1;91f82191efb8:37523 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/91f82191efb8:0, corePoolSize=1, maxPoolSize=1 2024-11-15T03:13:42,027 DEBUG [RS:2;91f82191efb8:41451 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/91f82191efb8:0, corePoolSize=1, maxPoolSize=1 2024-11-15T03:13:42,028 DEBUG [RS:0;91f82191efb8:42027 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/91f82191efb8:0, corePoolSize=1, maxPoolSize=1 2024-11-15T03:13:42,028 DEBUG [RS:1;91f82191efb8:37523 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/91f82191efb8:0, corePoolSize=1, maxPoolSize=1 2024-11-15T03:13:42,028 DEBUG [RS:0;91f82191efb8:42027 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/91f82191efb8:0, corePoolSize=1, maxPoolSize=1 2024-11-15T03:13:42,028 DEBUG [RS:1;91f82191efb8:37523 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/91f82191efb8:0, corePoolSize=1, maxPoolSize=1 2024-11-15T03:13:42,028 DEBUG [RS:2;91f82191efb8:41451 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/91f82191efb8:0, corePoolSize=1, maxPoolSize=1 2024-11-15T03:13:42,028 DEBUG [RS:0;91f82191efb8:42027 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/91f82191efb8:0, corePoolSize=1, maxPoolSize=1 2024-11-15T03:13:42,028 DEBUG [RS:1;91f82191efb8:37523 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/91f82191efb8:0, corePoolSize=1, maxPoolSize=1 2024-11-15T03:13:42,028 DEBUG [RS:2;91f82191efb8:41451 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/91f82191efb8:0, corePoolSize=1, maxPoolSize=1 2024-11-15T03:13:42,028 DEBUG [RS:0;91f82191efb8:42027 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/91f82191efb8:0, corePoolSize=1, maxPoolSize=1 2024-11-15T03:13:42,028 DEBUG [RS:1;91f82191efb8:37523 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/91f82191efb8:0, corePoolSize=3, maxPoolSize=3 2024-11-15T03:13:42,028 DEBUG [RS:0;91f82191efb8:42027 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/91f82191efb8:0, corePoolSize=1, maxPoolSize=1 2024-11-15T03:13:42,028 DEBUG [RS:2;91f82191efb8:41451 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/91f82191efb8:0, corePoolSize=1, maxPoolSize=1 2024-11-15T03:13:42,028 DEBUG [RS:1;91f82191efb8:37523 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/91f82191efb8:0, corePoolSize=3, maxPoolSize=3 2024-11-15T03:13:42,028 DEBUG [RS:0;91f82191efb8:42027 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/91f82191efb8:0, corePoolSize=3, maxPoolSize=3 2024-11-15T03:13:42,028 DEBUG [RS:2;91f82191efb8:41451 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/91f82191efb8:0, corePoolSize=1, maxPoolSize=1 2024-11-15T03:13:42,028 DEBUG [RS:0;91f82191efb8:42027 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/91f82191efb8:0, corePoolSize=3, maxPoolSize=3 2024-11-15T03:13:42,028 DEBUG [RS:2;91f82191efb8:41451 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/91f82191efb8:0, corePoolSize=1, maxPoolSize=1 2024-11-15T03:13:42,029 DEBUG [RS:2;91f82191efb8:41451 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/91f82191efb8:0, corePoolSize=3, maxPoolSize=3 2024-11-15T03:13:42,029 DEBUG [RS:2;91f82191efb8:41451 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/91f82191efb8:0, corePoolSize=3, maxPoolSize=3 2024-11-15T03:13:42,033 INFO [RS:1;91f82191efb8:37523 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T03:13:42,033 INFO [RS:1;91f82191efb8:37523 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T03:13:42,033 INFO [RS:1;91f82191efb8:37523 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T03:13:42,033 INFO [RS:1;91f82191efb8:37523 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-15T03:13:42,033 INFO [RS:1;91f82191efb8:37523 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-15T03:13:42,033 INFO [RS:1;91f82191efb8:37523 {}] hbase.ChoreService(168): Chore ScheduledChore name=91f82191efb8,37523,1731640420593-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-15T03:13:42,035 INFO [RS:0;91f82191efb8:42027 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T03:13:42,036 INFO [RS:0;91f82191efb8:42027 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T03:13:42,036 INFO [RS:0;91f82191efb8:42027 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T03:13:42,036 INFO [RS:0;91f82191efb8:42027 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-15T03:13:42,036 INFO [RS:0;91f82191efb8:42027 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-15T03:13:42,036 INFO [RS:0;91f82191efb8:42027 {}] hbase.ChoreService(168): Chore ScheduledChore name=91f82191efb8,42027,1731640420387-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-15T03:13:42,036 INFO [RS:2;91f82191efb8:41451 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T03:13:42,037 INFO [RS:2;91f82191efb8:41451 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T03:13:42,037 INFO [RS:2;91f82191efb8:41451 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T03:13:42,037 INFO [RS:2;91f82191efb8:41451 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-15T03:13:42,037 INFO [RS:2;91f82191efb8:41451 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-15T03:13:42,037 INFO [RS:2;91f82191efb8:41451 {}] hbase.ChoreService(168): Chore ScheduledChore name=91f82191efb8,41451,1731640420638-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-15T03:13:42,054 INFO [RS:0;91f82191efb8:42027 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-15T03:13:42,054 INFO [RS:1;91f82191efb8:37523 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-15T03:13:42,056 INFO [RS:1;91f82191efb8:37523 {}] hbase.ChoreService(168): Chore ScheduledChore name=91f82191efb8,37523,1731640420593-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T03:13:42,056 INFO [RS:0;91f82191efb8:42027 {}] hbase.ChoreService(168): Chore ScheduledChore name=91f82191efb8,42027,1731640420387-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T03:13:42,056 INFO [RS:0;91f82191efb8:42027 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T03:13:42,056 INFO [RS:1;91f82191efb8:37523 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T03:13:42,057 INFO [RS:0;91f82191efb8:42027 {}] regionserver.Replication(171): 91f82191efb8,42027,1731640420387 started 2024-11-15T03:13:42,057 INFO [RS:1;91f82191efb8:37523 {}] regionserver.Replication(171): 91f82191efb8,37523,1731640420593 started 2024-11-15T03:13:42,058 INFO [RS:2;91f82191efb8:41451 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-15T03:13:42,058 INFO [RS:2;91f82191efb8:41451 {}] hbase.ChoreService(168): Chore ScheduledChore name=91f82191efb8,41451,1731640420638-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T03:13:42,059 INFO [RS:2;91f82191efb8:41451 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T03:13:42,059 INFO [RS:2;91f82191efb8:41451 {}] regionserver.Replication(171): 91f82191efb8,41451,1731640420638 started 2024-11-15T03:13:42,074 INFO [RS:1;91f82191efb8:37523 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T03:13:42,074 INFO [RS:0;91f82191efb8:42027 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T03:13:42,074 INFO [RS:1;91f82191efb8:37523 {}] regionserver.HRegionServer(1482): Serving as 91f82191efb8,37523,1731640420593, RpcServer on 91f82191efb8/172.17.0.2:37523, sessionid=0x1013c7bdd830002 2024-11-15T03:13:42,074 INFO [RS:0;91f82191efb8:42027 {}] regionserver.HRegionServer(1482): Serving as 91f82191efb8,42027,1731640420387, RpcServer on 91f82191efb8/172.17.0.2:42027, sessionid=0x1013c7bdd830001 2024-11-15T03:13:42,075 DEBUG [RS:1;91f82191efb8:37523 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-15T03:13:42,075 DEBUG [RS:0;91f82191efb8:42027 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-15T03:13:42,075 DEBUG [RS:1;91f82191efb8:37523 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 91f82191efb8,37523,1731640420593 2024-11-15T03:13:42,075 DEBUG [RS:0;91f82191efb8:42027 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 91f82191efb8,42027,1731640420387 2024-11-15T03:13:42,075 DEBUG [RS:1;91f82191efb8:37523 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '91f82191efb8,37523,1731640420593' 2024-11-15T03:13:42,075 DEBUG [RS:0;91f82191efb8:42027 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '91f82191efb8,42027,1731640420387' 2024-11-15T03:13:42,075 DEBUG [RS:1;91f82191efb8:37523 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-15T03:13:42,076 DEBUG [RS:0;91f82191efb8:42027 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-15T03:13:42,076 DEBUG [RS:1;91f82191efb8:37523 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-15T03:13:42,076 DEBUG [RS:0;91f82191efb8:42027 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-15T03:13:42,077 DEBUG [RS:0;91f82191efb8:42027 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-15T03:13:42,077 DEBUG [RS:1;91f82191efb8:37523 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-15T03:13:42,077 DEBUG [RS:0;91f82191efb8:42027 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-15T03:13:42,077 DEBUG [RS:1;91f82191efb8:37523 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-15T03:13:42,077 DEBUG [RS:0;91f82191efb8:42027 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 91f82191efb8,42027,1731640420387 2024-11-15T03:13:42,077 DEBUG [RS:1;91f82191efb8:37523 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 91f82191efb8,37523,1731640420593 2024-11-15T03:13:42,077 DEBUG [RS:0;91f82191efb8:42027 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '91f82191efb8,42027,1731640420387' 2024-11-15T03:13:42,077 DEBUG [RS:1;91f82191efb8:37523 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '91f82191efb8,37523,1731640420593' 2024-11-15T03:13:42,077 DEBUG [RS:0;91f82191efb8:42027 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-15T03:13:42,077 DEBUG [RS:1;91f82191efb8:37523 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-15T03:13:42,078 DEBUG [RS:1;91f82191efb8:37523 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-15T03:13:42,078 DEBUG [RS:0;91f82191efb8:42027 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-15T03:13:42,078 DEBUG [RS:1;91f82191efb8:37523 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-15T03:13:42,078 DEBUG [RS:0;91f82191efb8:42027 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-15T03:13:42,078 INFO [RS:1;91f82191efb8:37523 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-15T03:13:42,078 INFO [RS:0;91f82191efb8:42027 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-15T03:13:42,078 INFO [RS:1;91f82191efb8:37523 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-15T03:13:42,078 INFO [RS:0;91f82191efb8:42027 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-15T03:13:42,080 INFO [RS:2;91f82191efb8:41451 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T03:13:42,081 INFO [RS:2;91f82191efb8:41451 {}] regionserver.HRegionServer(1482): Serving as 91f82191efb8,41451,1731640420638, RpcServer on 91f82191efb8/172.17.0.2:41451, sessionid=0x1013c7bdd830003 2024-11-15T03:13:42,081 DEBUG [RS:2;91f82191efb8:41451 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-15T03:13:42,081 DEBUG [RS:2;91f82191efb8:41451 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 91f82191efb8,41451,1731640420638 2024-11-15T03:13:42,081 DEBUG [RS:2;91f82191efb8:41451 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '91f82191efb8,41451,1731640420638' 2024-11-15T03:13:42,081 DEBUG [RS:2;91f82191efb8:41451 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-15T03:13:42,082 DEBUG [RS:2;91f82191efb8:41451 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-15T03:13:42,083 DEBUG [RS:2;91f82191efb8:41451 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-15T03:13:42,083 DEBUG [RS:2;91f82191efb8:41451 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-15T03:13:42,083 DEBUG [RS:2;91f82191efb8:41451 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 91f82191efb8,41451,1731640420638 2024-11-15T03:13:42,083 DEBUG [RS:2;91f82191efb8:41451 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '91f82191efb8,41451,1731640420638' 2024-11-15T03:13:42,083 DEBUG [RS:2;91f82191efb8:41451 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-15T03:13:42,084 DEBUG [RS:2;91f82191efb8:41451 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-15T03:13:42,084 DEBUG [RS:2;91f82191efb8:41451 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-15T03:13:42,084 INFO [RS:2;91f82191efb8:41451 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-15T03:13:42,084 INFO [RS:2;91f82191efb8:41451 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-15T03:13:42,107 WARN [91f82191efb8:42051 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-15T03:13:42,187 INFO [RS:2;91f82191efb8:41451 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-15T03:13:42,187 INFO [RS:1;91f82191efb8:37523 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-15T03:13:42,187 INFO [RS:0;91f82191efb8:42027 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-15T03:13:42,190 INFO [RS:0;91f82191efb8:42027 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=91f82191efb8%2C42027%2C1731640420387, suffix=, logDir=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/WALs/91f82191efb8,42027,1731640420387, archiveDir=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/oldWALs, maxLogs=32 2024-11-15T03:13:42,190 INFO [RS:2;91f82191efb8:41451 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=91f82191efb8%2C41451%2C1731640420638, suffix=, logDir=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/WALs/91f82191efb8,41451,1731640420638, archiveDir=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/oldWALs, maxLogs=32 2024-11-15T03:13:42,190 INFO [RS:1;91f82191efb8:37523 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=91f82191efb8%2C37523%2C1731640420593, suffix=, logDir=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/WALs/91f82191efb8,37523,1731640420593, archiveDir=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/oldWALs, maxLogs=32 2024-11-15T03:13:42,209 DEBUG [RS:2;91f82191efb8:41451 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/WALs/91f82191efb8,41451,1731640420638/91f82191efb8%2C41451%2C1731640420638.1731640422194, exclude list is [], retry=0 2024-11-15T03:13:42,213 DEBUG [RS:1;91f82191efb8:37523 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/WALs/91f82191efb8,37523,1731640420593/91f82191efb8%2C37523%2C1731640420593.1731640422194, exclude list is [], retry=0 2024-11-15T03:13:42,213 DEBUG [RS:0;91f82191efb8:42027 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/WALs/91f82191efb8,42027,1731640420387/91f82191efb8%2C42027%2C1731640420387.1731640422194, exclude list is [], retry=0 2024-11-15T03:13:42,213 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36559,DS-818fe711-350c-400c-9163-61e8ae4f0904,DISK] 2024-11-15T03:13:42,213 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38319,DS-e648626c-dc65-44f2-ab4f-9b6acd52971a,DISK] 2024-11-15T03:13:42,214 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46785,DS-0b930937-9ceb-4d09-87e5-60bb062fe96b,DISK] 2024-11-15T03:13:42,218 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38319,DS-e648626c-dc65-44f2-ab4f-9b6acd52971a,DISK] 2024-11-15T03:13:42,218 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36559,DS-818fe711-350c-400c-9163-61e8ae4f0904,DISK] 2024-11-15T03:13:42,218 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38319,DS-e648626c-dc65-44f2-ab4f-9b6acd52971a,DISK] 2024-11-15T03:13:42,218 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46785,DS-0b930937-9ceb-4d09-87e5-60bb062fe96b,DISK] 2024-11-15T03:13:42,218 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36559,DS-818fe711-350c-400c-9163-61e8ae4f0904,DISK] 2024-11-15T03:13:42,218 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46785,DS-0b930937-9ceb-4d09-87e5-60bb062fe96b,DISK] 2024-11-15T03:13:42,220 INFO [RS:2;91f82191efb8:41451 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/WALs/91f82191efb8,41451,1731640420638/91f82191efb8%2C41451%2C1731640420638.1731640422194 2024-11-15T03:13:42,223 DEBUG [RS:2;91f82191efb8:41451 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37901:37901),(127.0.0.1/127.0.0.1:45153:45153),(127.0.0.1/127.0.0.1:36035:36035)] 2024-11-15T03:13:42,229 INFO [RS:1;91f82191efb8:37523 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/WALs/91f82191efb8,37523,1731640420593/91f82191efb8%2C37523%2C1731640420593.1731640422194 2024-11-15T03:13:42,232 DEBUG [RS:1;91f82191efb8:37523 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37901:37901),(127.0.0.1/127.0.0.1:45153:45153),(127.0.0.1/127.0.0.1:36035:36035)] 2024-11-15T03:13:42,234 INFO [RS:0;91f82191efb8:42027 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/WALs/91f82191efb8,42027,1731640420387/91f82191efb8%2C42027%2C1731640420387.1731640422194 2024-11-15T03:13:42,234 DEBUG [RS:0;91f82191efb8:42027 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37901:37901),(127.0.0.1/127.0.0.1:45153:45153),(127.0.0.1/127.0.0.1:36035:36035)] 2024-11-15T03:13:42,360 DEBUG [91f82191efb8:42051 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-15T03:13:42,369 DEBUG [91f82191efb8:42051 {}] balancer.BalancerClusterState(204): Hosts are {91f82191efb8=0} racks are {/default-rack=0} 2024-11-15T03:13:42,374 DEBUG [91f82191efb8:42051 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-15T03:13:42,374 DEBUG [91f82191efb8:42051 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-15T03:13:42,375 DEBUG [91f82191efb8:42051 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-15T03:13:42,375 DEBUG [91f82191efb8:42051 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-15T03:13:42,375 DEBUG [91f82191efb8:42051 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-15T03:13:42,375 DEBUG [91f82191efb8:42051 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-15T03:13:42,375 INFO [91f82191efb8:42051 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-15T03:13:42,375 INFO [91f82191efb8:42051 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-15T03:13:42,375 INFO [91f82191efb8:42051 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-15T03:13:42,375 DEBUG [91f82191efb8:42051 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-15T03:13:42,381 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=91f82191efb8,42027,1731640420387 2024-11-15T03:13:42,388 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 91f82191efb8,42027,1731640420387, state=OPENING 2024-11-15T03:13:42,409 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-15T03:13:42,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37523-0x1013c7bdd830002, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T03:13:42,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41451-0x1013c7bdd830003, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T03:13:42,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42051-0x1013c7bdd830000, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T03:13:42,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42027-0x1013c7bdd830001, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T03:13:42,506 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:42,507 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:42,507 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:42,507 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:42,510 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T03:13:42,513 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=91f82191efb8,42027,1731640420387}] 2024-11-15T03:13:42,690 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-15T03:13:42,692 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37185, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-15T03:13:42,704 INFO [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-15T03:13:42,704 INFO [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T03:13:42,705 INFO [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-15T03:13:42,708 INFO [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=91f82191efb8%2C42027%2C1731640420387.meta, suffix=.meta, logDir=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/WALs/91f82191efb8,42027,1731640420387, archiveDir=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/oldWALs, maxLogs=32 2024-11-15T03:13:42,724 DEBUG [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/WALs/91f82191efb8,42027,1731640420387/91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, exclude list is [], retry=0 2024-11-15T03:13:42,728 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46785,DS-0b930937-9ceb-4d09-87e5-60bb062fe96b,DISK] 2024-11-15T03:13:42,728 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36559,DS-818fe711-350c-400c-9163-61e8ae4f0904,DISK] 2024-11-15T03:13:42,728 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38319,DS-e648626c-dc65-44f2-ab4f-9b6acd52971a,DISK] 2024-11-15T03:13:42,731 INFO [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/WALs/91f82191efb8,42027,1731640420387/91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta 2024-11-15T03:13:42,731 DEBUG [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36035:36035),(127.0.0.1/127.0.0.1:45153:45153),(127.0.0.1/127.0.0.1:37901:37901)] 2024-11-15T03:13:42,732 DEBUG [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-15T03:13:42,733 DEBUG [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-15T03:13:42,735 DEBUG [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-15T03:13:42,740 INFO [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-15T03:13:42,743 DEBUG [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-15T03:13:42,744 DEBUG [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T03:13:42,744 DEBUG [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-15T03:13:42,744 DEBUG [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-15T03:13:42,747 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-15T03:13:42,749 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-15T03:13:42,749 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T03:13:42,750 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T03:13:42,750 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-15T03:13:42,752 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-15T03:13:42,752 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T03:13:42,753 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T03:13:42,754 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-15T03:13:42,755 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-15T03:13:42,755 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T03:13:42,756 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T03:13:42,756 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-15T03:13:42,758 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-15T03:13:42,758 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T03:13:42,759 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T03:13:42,759 DEBUG [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-15T03:13:42,760 DEBUG [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/hbase/meta/1588230740 2024-11-15T03:13:42,763 DEBUG [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/hbase/meta/1588230740 2024-11-15T03:13:42,766 DEBUG [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-15T03:13:42,767 DEBUG [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-15T03:13:42,768 DEBUG [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-15T03:13:42,771 DEBUG [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-15T03:13:42,772 INFO [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74622857, jitterRate=0.11196722090244293}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-15T03:13:42,772 DEBUG [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-15T03:13:42,773 DEBUG [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731640422745Writing region info on filesystem at 1731640422745Initializing all the Stores at 1731640422747 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731640422747Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731640422747Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731640422747Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731640422747Cleaning up temporary data from old regions at 1731640422767 (+20 ms)Running coprocessor post-open hooks at 1731640422772 (+5 ms)Region opened successfully at 1731640422773 (+1 ms) 2024-11-15T03:13:42,779 INFO [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731640422681 2024-11-15T03:13:42,789 DEBUG [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-15T03:13:42,790 INFO [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-15T03:13:42,791 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=91f82191efb8,42027,1731640420387 2024-11-15T03:13:42,794 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 91f82191efb8,42027,1731640420387, state=OPEN 2024-11-15T03:13:42,804 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42027-0x1013c7bdd830001, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:42,804 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42051-0x1013c7bdd830000, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:42,804 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37523-0x1013c7bdd830002, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:42,804 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41451-0x1013c7bdd830003, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:42,804 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:42,804 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:42,804 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:42,804 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:42,805 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=91f82191efb8,42027,1731640420387 2024-11-15T03:13:42,810 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-15T03:13:42,810 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=91f82191efb8,42027,1731640420387 in 293 msec 2024-11-15T03:13:42,818 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-15T03:13:42,818 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 871 msec 2024-11-15T03:13:42,820 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-15T03:13:42,820 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-15T03:13:42,837 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T03:13:42,838 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=91f82191efb8,42027,1731640420387, seqNum=-1] 2024-11-15T03:13:42,864 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T03:13:42,866 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60167, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T03:13:42,888 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1650 sec 2024-11-15T03:13:42,888 INFO [master/91f82191efb8:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731640422888, completionTime=-1 2024-11-15T03:13:42,890 INFO [master/91f82191efb8:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-15T03:13:42,891 DEBUG [master/91f82191efb8:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-15T03:13:42,913 INFO [master/91f82191efb8:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=3 2024-11-15T03:13:42,913 INFO [master/91f82191efb8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731640482913 2024-11-15T03:13:42,913 INFO [master/91f82191efb8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731640542913 2024-11-15T03:13:42,913 INFO [master/91f82191efb8:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 22 msec 2024-11-15T03:13:42,915 DEBUG [master/91f82191efb8:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-15T03:13:42,921 INFO [master/91f82191efb8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=91f82191efb8,42051,1731640419585-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T03:13:42,922 INFO [master/91f82191efb8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=91f82191efb8,42051,1731640419585-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T03:13:42,922 INFO [master/91f82191efb8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=91f82191efb8,42051,1731640419585-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T03:13:42,923 INFO [master/91f82191efb8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-91f82191efb8:42051, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T03:13:42,923 INFO [master/91f82191efb8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-15T03:13:42,924 INFO [master/91f82191efb8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-15T03:13:42,930 DEBUG [master/91f82191efb8:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-15T03:13:42,949 INFO [master/91f82191efb8:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.200sec 2024-11-15T03:13:42,951 INFO [master/91f82191efb8:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-15T03:13:42,952 INFO [master/91f82191efb8:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-15T03:13:42,953 INFO [master/91f82191efb8:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-15T03:13:42,954 INFO [master/91f82191efb8:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-15T03:13:42,954 INFO [master/91f82191efb8:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-15T03:13:42,954 INFO [master/91f82191efb8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=91f82191efb8,42051,1731640419585-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-15T03:13:42,955 INFO [master/91f82191efb8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=91f82191efb8,42051,1731640419585-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-15T03:13:42,959 DEBUG [master/91f82191efb8:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-15T03:13:42,960 INFO [master/91f82191efb8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-15T03:13:42,960 INFO [master/91f82191efb8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=91f82191efb8,42051,1731640419585-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T03:13:42,977 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1482df6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T03:13:42,981 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-15T03:13:42,981 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-15T03:13:42,986 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 91f82191efb8,42051,-1 for getting cluster id 2024-11-15T03:13:42,988 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T03:13:42,996 DEBUG [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '451b97df-43ba-4345-aef9-5c28b9dc1656' 2024-11-15T03:13:42,998 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T03:13:42,999 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "451b97df-43ba-4345-aef9-5c28b9dc1656" 2024-11-15T03:13:42,999 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45d3dec2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T03:13:42,999 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [91f82191efb8,42051,-1] 2024-11-15T03:13:43,001 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T03:13:43,003 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T03:13:43,004 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50820, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T03:13:43,007 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bf9209, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T03:13:43,007 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T03:13:43,014 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=91f82191efb8,42027,1731640420387, seqNum=-1] 2024-11-15T03:13:43,014 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T03:13:43,017 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51078, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T03:13:43,052 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=91f82191efb8,42051,1731640419585 2024-11-15T03:13:43,056 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T03:13:43,061 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 91f82191efb8,42051,1731640419585 2024-11-15T03:13:43,063 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@28dae60e 2024-11-15T03:13:43,064 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T03:13:43,067 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50836, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T03:13:43,074 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42051 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'test', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T03:13:43,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42051 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test 2024-11-15T03:13:43,088 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T03:13:43,090 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42051 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "test" procId is: 4 2024-11-15T03:13:43,091 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T03:13:43,094 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T03:13:43,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T03:13:43,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46785 is added to blk_1073741837_1013 (size=358) 2024-11-15T03:13:43,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36559 is added to blk_1073741837_1013 (size=358) 2024-11-15T03:13:43,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38319 is added to blk_1073741837_1013 (size=358) 2024-11-15T03:13:43,111 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5965b596ce7b87fabde47569d60c3f95, NAME => 'test,,1731640423069.5965b596ce7b87fabde47569d60c3f95.', STARTKEY => '', ENDKEY => 'bbb'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46 2024-11-15T03:13:43,111 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 5971f307349447e7f24b0dd05b96cffc, NAME => 'test,ccc,1731640423069.5971f307349447e7f24b0dd05b96cffc.', STARTKEY => 'ccc', ENDKEY => 'ddd'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46 2024-11-15T03:13:43,112 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 6130297c45bd23340b368de2cc708c90, NAME => 'test,bbb,1731640423069.6130297c45bd23340b368de2cc708c90.', STARTKEY => 'bbb', ENDKEY => 'ccc'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46 2024-11-15T03:13:43,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38319 is added to blk_1073741838_1014 (size=45) 2024-11-15T03:13:43,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46785 is added to blk_1073741838_1014 (size=45) 2024-11-15T03:13:43,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36559 is added to blk_1073741838_1014 (size=45) 2024-11-15T03:13:43,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38319 is added to blk_1073741840_1016 (size=45) 2024-11-15T03:13:43,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46785 is added to blk_1073741839_1015 (size=42) 2024-11-15T03:13:43,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36559 is added to blk_1073741839_1015 (size=42) 2024-11-15T03:13:43,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36559 is added to blk_1073741840_1016 (size=45) 2024-11-15T03:13:43,134 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,bbb,1731640423069.6130297c45bd23340b368de2cc708c90.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T03:13:43,134 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ccc,1731640423069.5971f307349447e7f24b0dd05b96cffc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T03:13:43,134 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 6130297c45bd23340b368de2cc708c90, disabling compactions & flushes 2024-11-15T03:13:43,134 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,bbb,1731640423069.6130297c45bd23340b368de2cc708c90. 2024-11-15T03:13:43,134 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 5971f307349447e7f24b0dd05b96cffc, disabling compactions & flushes 2024-11-15T03:13:43,134 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1731640423069.6130297c45bd23340b368de2cc708c90. 2024-11-15T03:13:43,134 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ccc,1731640423069.5971f307349447e7f24b0dd05b96cffc. 2024-11-15T03:13:43,134 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1731640423069.5971f307349447e7f24b0dd05b96cffc. 2024-11-15T03:13:43,134 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1731640423069.6130297c45bd23340b368de2cc708c90. after waiting 0 ms 2024-11-15T03:13:43,134 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1731640423069.5971f307349447e7f24b0dd05b96cffc. after waiting 0 ms 2024-11-15T03:13:43,134 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1731640423069.6130297c45bd23340b368de2cc708c90. 2024-11-15T03:13:43,134 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1731640423069.5971f307349447e7f24b0dd05b96cffc. 2024-11-15T03:13:43,134 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,bbb,1731640423069.6130297c45bd23340b368de2cc708c90. 2024-11-15T03:13:43,134 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ccc,1731640423069.5971f307349447e7f24b0dd05b96cffc. 2024-11-15T03:13:43,134 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 6130297c45bd23340b368de2cc708c90: Waiting for close lock at 1731640423134Disabling compacts and flushes for region at 1731640423134Disabling writes for close at 1731640423134Writing region close event to WAL at 1731640423134Closed at 1731640423134 2024-11-15T03:13:43,134 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 5971f307349447e7f24b0dd05b96cffc: Waiting for close lock at 1731640423134Disabling compacts and flushes for region at 1731640423134Disabling writes for close at 1731640423134Writing region close event to WAL at 1731640423134Closed at 1731640423134 2024-11-15T03:13:43,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46785 is added to blk_1073741840_1016 (size=45) 2024-11-15T03:13:43,135 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => e0f9e1e28ca8906f0f82f959a60c55ce, NAME => 'test,eee,1731640423069.e0f9e1e28ca8906f0f82f959a60c55ce.', STARTKEY => 'eee', ENDKEY => 'fff'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46 2024-11-15T03:13:43,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38319 is added to blk_1073741839_1015 (size=42) 2024-11-15T03:13:43,136 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 1beb579fcf8e161a0b9c2051c671a8f2, NAME => 'test,ddd,1731640423069.1beb579fcf8e161a0b9c2051c671a8f2.', STARTKEY => 'ddd', ENDKEY => 'eee'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46 2024-11-15T03:13:43,136 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,,1731640423069.5965b596ce7b87fabde47569d60c3f95.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T03:13:43,136 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 5965b596ce7b87fabde47569d60c3f95, disabling compactions & flushes 2024-11-15T03:13:43,136 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,,1731640423069.5965b596ce7b87fabde47569d60c3f95. 2024-11-15T03:13:43,136 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1731640423069.5965b596ce7b87fabde47569d60c3f95. 2024-11-15T03:13:43,137 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,,1731640423069.5965b596ce7b87fabde47569d60c3f95. after waiting 0 ms 2024-11-15T03:13:43,137 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,,1731640423069.5965b596ce7b87fabde47569d60c3f95. 2024-11-15T03:13:43,137 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,,1731640423069.5965b596ce7b87fabde47569d60c3f95. 2024-11-15T03:13:43,137 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5965b596ce7b87fabde47569d60c3f95: Waiting for close lock at 1731640423136Disabling compacts and flushes for region at 1731640423136Disabling writes for close at 1731640423137 (+1 ms)Writing region close event to WAL at 1731640423137Closed at 1731640423137 2024-11-15T03:13:43,137 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c4b611da4a73d4cfc99379713486e21c, NAME => 'test,fff,1731640423069.c4b611da4a73d4cfc99379713486e21c.', STARTKEY => 'fff', ENDKEY => 'ggg'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46 2024-11-15T03:13:43,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46785 is added to blk_1073741843_1019 (size=45) 2024-11-15T03:13:43,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38319 is added to blk_1073741843_1019 (size=45) 2024-11-15T03:13:43,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36559 is added to blk_1073741843_1019 (size=45) 2024-11-15T03:13:43,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38319 is added to blk_1073741842_1018 (size=45) 2024-11-15T03:13:43,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46785 is added to blk_1073741842_1018 (size=45) 2024-11-15T03:13:43,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36559 is added to blk_1073741842_1018 (size=45) 2024-11-15T03:13:43,153 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ddd,1731640423069.1beb579fcf8e161a0b9c2051c671a8f2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T03:13:43,153 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,fff,1731640423069.c4b611da4a73d4cfc99379713486e21c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T03:13:43,154 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing c4b611da4a73d4cfc99379713486e21c, disabling compactions & flushes 2024-11-15T03:13:43,154 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 1beb579fcf8e161a0b9c2051c671a8f2, disabling compactions & flushes 2024-11-15T03:13:43,154 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,fff,1731640423069.c4b611da4a73d4cfc99379713486e21c. 2024-11-15T03:13:43,154 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ddd,1731640423069.1beb579fcf8e161a0b9c2051c671a8f2. 2024-11-15T03:13:43,154 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1731640423069.c4b611da4a73d4cfc99379713486e21c. 2024-11-15T03:13:43,154 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1731640423069.1beb579fcf8e161a0b9c2051c671a8f2. 2024-11-15T03:13:43,154 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,fff,1731640423069.c4b611da4a73d4cfc99379713486e21c. after waiting 0 ms 2024-11-15T03:13:43,154 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1731640423069.1beb579fcf8e161a0b9c2051c671a8f2. after waiting 0 ms 2024-11-15T03:13:43,154 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,fff,1731640423069.c4b611da4a73d4cfc99379713486e21c. 2024-11-15T03:13:43,154 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1731640423069.1beb579fcf8e161a0b9c2051c671a8f2. 2024-11-15T03:13:43,154 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,fff,1731640423069.c4b611da4a73d4cfc99379713486e21c. 2024-11-15T03:13:43,154 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ddd,1731640423069.1beb579fcf8e161a0b9c2051c671a8f2. 2024-11-15T03:13:43,154 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for c4b611da4a73d4cfc99379713486e21c: Waiting for close lock at 1731640423153Disabling compacts and flushes for region at 1731640423153Disabling writes for close at 1731640423154 (+1 ms)Writing region close event to WAL at 1731640423154Closed at 1731640423154 2024-11-15T03:13:43,154 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 1beb579fcf8e161a0b9c2051c671a8f2: Waiting for close lock at 1731640423153Disabling compacts and flushes for region at 1731640423153Disabling writes for close at 1731640423154 (+1 ms)Writing region close event to WAL at 1731640423154Closed at 1731640423154 2024-11-15T03:13:43,155 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d0eff1b91bc9b77579a5c44f1c985985, NAME => 'test,ggg,1731640423069.d0eff1b91bc9b77579a5c44f1c985985.', STARTKEY => 'ggg', ENDKEY => 'hhh'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46 2024-11-15T03:13:43,155 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 48ff514f5498660a4e8fc753340bb847, NAME => 'test,hhh,1731640423069.48ff514f5498660a4e8fc753340bb847.', STARTKEY => 'hhh', ENDKEY => 'iii'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46 2024-11-15T03:13:43,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46785 is added to blk_1073741841_1017 (size=45) 2024-11-15T03:13:43,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36559 is added to blk_1073741841_1017 (size=45) 2024-11-15T03:13:43,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38319 is added to blk_1073741841_1017 (size=45) 2024-11-15T03:13:43,161 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,eee,1731640423069.e0f9e1e28ca8906f0f82f959a60c55ce.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T03:13:43,161 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing e0f9e1e28ca8906f0f82f959a60c55ce, disabling compactions & flushes 2024-11-15T03:13:43,161 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,eee,1731640423069.e0f9e1e28ca8906f0f82f959a60c55ce. 2024-11-15T03:13:43,161 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1731640423069.e0f9e1e28ca8906f0f82f959a60c55ce. 2024-11-15T03:13:43,161 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,eee,1731640423069.e0f9e1e28ca8906f0f82f959a60c55ce. after waiting 0 ms 2024-11-15T03:13:43,161 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,eee,1731640423069.e0f9e1e28ca8906f0f82f959a60c55ce. 2024-11-15T03:13:43,161 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,eee,1731640423069.e0f9e1e28ca8906f0f82f959a60c55ce. 2024-11-15T03:13:43,162 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for e0f9e1e28ca8906f0f82f959a60c55ce: Waiting for close lock at 1731640423161Disabling compacts and flushes for region at 1731640423161Disabling writes for close at 1731640423161Writing region close event to WAL at 1731640423161Closed at 1731640423161 2024-11-15T03:13:43,162 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => f5219cfe492c74eb8312930d328964b6, NAME => 'test,iii,1731640423069.f5219cfe492c74eb8312930d328964b6.', STARTKEY => 'iii', ENDKEY => 'jjj'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46 2024-11-15T03:13:43,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46785 is added to blk_1073741844_1020 (size=45) 2024-11-15T03:13:43,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38319 is added to blk_1073741844_1020 (size=45) 2024-11-15T03:13:43,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36559 is added to blk_1073741844_1020 (size=45) 2024-11-15T03:13:43,175 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,hhh,1731640423069.48ff514f5498660a4e8fc753340bb847.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T03:13:43,176 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 48ff514f5498660a4e8fc753340bb847, disabling compactions & flushes 2024-11-15T03:13:43,176 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,hhh,1731640423069.48ff514f5498660a4e8fc753340bb847. 2024-11-15T03:13:43,176 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1731640423069.48ff514f5498660a4e8fc753340bb847. 2024-11-15T03:13:43,176 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1731640423069.48ff514f5498660a4e8fc753340bb847. after waiting 0 ms 2024-11-15T03:13:43,176 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1731640423069.48ff514f5498660a4e8fc753340bb847. 2024-11-15T03:13:43,176 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,hhh,1731640423069.48ff514f5498660a4e8fc753340bb847. 2024-11-15T03:13:43,176 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 48ff514f5498660a4e8fc753340bb847: Waiting for close lock at 1731640423176Disabling compacts and flushes for region at 1731640423176Disabling writes for close at 1731640423176Writing region close event to WAL at 1731640423176Closed at 1731640423176 2024-11-15T03:13:43,176 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => b1704275babe02e36b29cfaadf5b81af, NAME => 'test,jjj,1731640423069.b1704275babe02e36b29cfaadf5b81af.', STARTKEY => 'jjj', ENDKEY => 'kkk'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46 2024-11-15T03:13:43,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38319 is added to blk_1073741845_1021 (size=45) 2024-11-15T03:13:43,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36559 is added to blk_1073741845_1021 (size=45) 2024-11-15T03:13:43,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38319 is added to blk_1073741846_1022 (size=45) 2024-11-15T03:13:43,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46785 is added to blk_1073741845_1021 (size=45) 2024-11-15T03:13:43,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36559 is added to blk_1073741846_1022 (size=45) 2024-11-15T03:13:43,186 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ggg,1731640423069.d0eff1b91bc9b77579a5c44f1c985985.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T03:13:43,186 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing d0eff1b91bc9b77579a5c44f1c985985, disabling compactions & flushes 2024-11-15T03:13:43,186 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ggg,1731640423069.d0eff1b91bc9b77579a5c44f1c985985. 2024-11-15T03:13:43,186 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1731640423069.d0eff1b91bc9b77579a5c44f1c985985. 2024-11-15T03:13:43,186 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1731640423069.d0eff1b91bc9b77579a5c44f1c985985. after waiting 0 ms 2024-11-15T03:13:43,186 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1731640423069.d0eff1b91bc9b77579a5c44f1c985985. 2024-11-15T03:13:43,186 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ggg,1731640423069.d0eff1b91bc9b77579a5c44f1c985985. 2024-11-15T03:13:43,187 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for d0eff1b91bc9b77579a5c44f1c985985: Waiting for close lock at 1731640423186Disabling compacts and flushes for region at 1731640423186Disabling writes for close at 1731640423186Writing region close event to WAL at 1731640423186Closed at 1731640423186 2024-11-15T03:13:43,187 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => bb2ead3b463edc0f2b603da7d19f1201, NAME => 'test,kkk,1731640423069.bb2ead3b463edc0f2b603da7d19f1201.', STARTKEY => 'kkk', ENDKEY => 'lll'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46 2024-11-15T03:13:43,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46785 is added to blk_1073741846_1022 (size=45) 2024-11-15T03:13:43,191 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,iii,1731640423069.f5219cfe492c74eb8312930d328964b6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T03:13:43,191 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing f5219cfe492c74eb8312930d328964b6, disabling compactions & flushes 2024-11-15T03:13:43,191 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,iii,1731640423069.f5219cfe492c74eb8312930d328964b6. 2024-11-15T03:13:43,191 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1731640423069.f5219cfe492c74eb8312930d328964b6. 2024-11-15T03:13:43,191 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,iii,1731640423069.f5219cfe492c74eb8312930d328964b6. after waiting 0 ms 2024-11-15T03:13:43,191 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,iii,1731640423069.f5219cfe492c74eb8312930d328964b6. 2024-11-15T03:13:43,191 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,iii,1731640423069.f5219cfe492c74eb8312930d328964b6. 2024-11-15T03:13:43,191 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for f5219cfe492c74eb8312930d328964b6: Waiting for close lock at 1731640423191Disabling compacts and flushes for region at 1731640423191Disabling writes for close at 1731640423191Writing region close event to WAL at 1731640423191Closed at 1731640423191 2024-11-15T03:13:43,192 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 860df8edb65f2fdd3c8bc00f89f94ba0, NAME => 'test,lll,1731640423069.860df8edb65f2fdd3c8bc00f89f94ba0.', STARTKEY => 'lll', ENDKEY => 'mmm'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46 2024-11-15T03:13:43,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38319 is added to blk_1073741847_1023 (size=45) 2024-11-15T03:13:43,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36559 is added to blk_1073741847_1023 (size=45) 2024-11-15T03:13:43,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46785 is added to blk_1073741847_1023 (size=45) 2024-11-15T03:13:43,201 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,jjj,1731640423069.b1704275babe02e36b29cfaadf5b81af.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T03:13:43,201 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing b1704275babe02e36b29cfaadf5b81af, disabling compactions & flushes 2024-11-15T03:13:43,201 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,jjj,1731640423069.b1704275babe02e36b29cfaadf5b81af. 2024-11-15T03:13:43,201 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1731640423069.b1704275babe02e36b29cfaadf5b81af. 2024-11-15T03:13:43,201 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1731640423069.b1704275babe02e36b29cfaadf5b81af. after waiting 0 ms 2024-11-15T03:13:43,201 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1731640423069.b1704275babe02e36b29cfaadf5b81af. 2024-11-15T03:13:43,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T03:13:43,201 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,jjj,1731640423069.b1704275babe02e36b29cfaadf5b81af. 2024-11-15T03:13:43,201 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for b1704275babe02e36b29cfaadf5b81af: Waiting for close lock at 1731640423201Disabling compacts and flushes for region at 1731640423201Disabling writes for close at 1731640423201Writing region close event to WAL at 1731640423201Closed at 1731640423201 2024-11-15T03:13:43,202 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => eaf85cc654192e46d3ea0904493eea12, NAME => 'test,mmm,1731640423069.eaf85cc654192e46d3ea0904493eea12.', STARTKEY => 'mmm', ENDKEY => 'nnn'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46 2024-11-15T03:13:43,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36559 is added to blk_1073741848_1024 (size=45) 2024-11-15T03:13:43,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46785 is added to blk_1073741848_1024 (size=45) 2024-11-15T03:13:43,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38319 is added to blk_1073741848_1024 (size=45) 2024-11-15T03:13:43,212 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,kkk,1731640423069.bb2ead3b463edc0f2b603da7d19f1201.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T03:13:43,213 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing bb2ead3b463edc0f2b603da7d19f1201, disabling compactions & flushes 2024-11-15T03:13:43,213 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,kkk,1731640423069.bb2ead3b463edc0f2b603da7d19f1201. 2024-11-15T03:13:43,213 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1731640423069.bb2ead3b463edc0f2b603da7d19f1201. 2024-11-15T03:13:43,213 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1731640423069.bb2ead3b463edc0f2b603da7d19f1201. after waiting 0 ms 2024-11-15T03:13:43,213 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1731640423069.bb2ead3b463edc0f2b603da7d19f1201. 2024-11-15T03:13:43,213 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,kkk,1731640423069.bb2ead3b463edc0f2b603da7d19f1201. 2024-11-15T03:13:43,213 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for bb2ead3b463edc0f2b603da7d19f1201: Waiting for close lock at 1731640423213Disabling compacts and flushes for region at 1731640423213Disabling writes for close at 1731640423213Writing region close event to WAL at 1731640423213Closed at 1731640423213 2024-11-15T03:13:43,214 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a73d2d096e2f14a7e9365ac762927d55, NAME => 'test,nnn,1731640423069.a73d2d096e2f14a7e9365ac762927d55.', STARTKEY => 'nnn', ENDKEY => 'ooo'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46 2024-11-15T03:13:43,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36559 is added to blk_1073741849_1025 (size=45) 2024-11-15T03:13:43,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38319 is added to blk_1073741849_1025 (size=45) 2024-11-15T03:13:43,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46785 is added to blk_1073741849_1025 (size=45) 2024-11-15T03:13:43,223 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,lll,1731640423069.860df8edb65f2fdd3c8bc00f89f94ba0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T03:13:43,223 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 860df8edb65f2fdd3c8bc00f89f94ba0, disabling compactions & flushes 2024-11-15T03:13:43,223 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,lll,1731640423069.860df8edb65f2fdd3c8bc00f89f94ba0. 2024-11-15T03:13:43,223 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1731640423069.860df8edb65f2fdd3c8bc00f89f94ba0. 2024-11-15T03:13:43,224 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,lll,1731640423069.860df8edb65f2fdd3c8bc00f89f94ba0. after waiting 0 ms 2024-11-15T03:13:43,224 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,lll,1731640423069.860df8edb65f2fdd3c8bc00f89f94ba0. 2024-11-15T03:13:43,224 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,lll,1731640423069.860df8edb65f2fdd3c8bc00f89f94ba0. 2024-11-15T03:13:43,224 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 860df8edb65f2fdd3c8bc00f89f94ba0: Waiting for close lock at 1731640423223Disabling compacts and flushes for region at 1731640423223Disabling writes for close at 1731640423224 (+1 ms)Writing region close event to WAL at 1731640423224Closed at 1731640423224 2024-11-15T03:13:43,224 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => b96380a8d7110dffa8cd7844252025dd, NAME => 'test,ooo,1731640423069.b96380a8d7110dffa8cd7844252025dd.', STARTKEY => 'ooo', ENDKEY => 'ppp'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46 2024-11-15T03:13:43,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38319 is added to blk_1073741850_1026 (size=45) 2024-11-15T03:13:43,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46785 is added to blk_1073741850_1026 (size=45) 2024-11-15T03:13:43,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36559 is added to blk_1073741850_1026 (size=45) 2024-11-15T03:13:43,233 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,mmm,1731640423069.eaf85cc654192e46d3ea0904493eea12.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T03:13:43,233 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing eaf85cc654192e46d3ea0904493eea12, disabling compactions & flushes 2024-11-15T03:13:43,233 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,mmm,1731640423069.eaf85cc654192e46d3ea0904493eea12. 2024-11-15T03:13:43,233 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1731640423069.eaf85cc654192e46d3ea0904493eea12. 2024-11-15T03:13:43,233 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1731640423069.eaf85cc654192e46d3ea0904493eea12. after waiting 0 ms 2024-11-15T03:13:43,233 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1731640423069.eaf85cc654192e46d3ea0904493eea12. 2024-11-15T03:13:43,233 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,mmm,1731640423069.eaf85cc654192e46d3ea0904493eea12. 2024-11-15T03:13:43,233 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for eaf85cc654192e46d3ea0904493eea12: Waiting for close lock at 1731640423233Disabling compacts and flushes for region at 1731640423233Disabling writes for close at 1731640423233Writing region close event to WAL at 1731640423233Closed at 1731640423233 2024-11-15T03:13:43,234 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => fac2a1b464a5ff2d581af6baef327bde, NAME => 'test,ppp,1731640423069.fac2a1b464a5ff2d581af6baef327bde.', STARTKEY => 'ppp', ENDKEY => 'qqq'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46 2024-11-15T03:13:43,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36559 is added to blk_1073741851_1027 (size=45) 2024-11-15T03:13:43,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46785 is added to blk_1073741851_1027 (size=45) 2024-11-15T03:13:43,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38319 is added to blk_1073741851_1027 (size=45) 2024-11-15T03:13:43,242 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,nnn,1731640423069.a73d2d096e2f14a7e9365ac762927d55.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T03:13:43,242 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing a73d2d096e2f14a7e9365ac762927d55, disabling compactions & flushes 2024-11-15T03:13:43,242 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,nnn,1731640423069.a73d2d096e2f14a7e9365ac762927d55. 2024-11-15T03:13:43,242 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1731640423069.a73d2d096e2f14a7e9365ac762927d55. 2024-11-15T03:13:43,242 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1731640423069.a73d2d096e2f14a7e9365ac762927d55. after waiting 0 ms 2024-11-15T03:13:43,242 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1731640423069.a73d2d096e2f14a7e9365ac762927d55. 2024-11-15T03:13:43,242 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,nnn,1731640423069.a73d2d096e2f14a7e9365ac762927d55. 2024-11-15T03:13:43,242 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for a73d2d096e2f14a7e9365ac762927d55: Waiting for close lock at 1731640423242Disabling compacts and flushes for region at 1731640423242Disabling writes for close at 1731640423242Writing region close event to WAL at 1731640423242Closed at 1731640423242 2024-11-15T03:13:43,243 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 23f804d67a47ba185ffd6f6bc7b75464, NAME => 'test,qqq,1731640423069.23f804d67a47ba185ffd6f6bc7b75464.', STARTKEY => 'qqq', ENDKEY => 'rrr'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46 2024-11-15T03:13:43,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46785 is added to blk_1073741852_1028 (size=45) 2024-11-15T03:13:43,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38319 is added to blk_1073741852_1028 (size=45) 2024-11-15T03:13:43,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36559 is added to blk_1073741852_1028 (size=45) 2024-11-15T03:13:43,251 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ooo,1731640423069.b96380a8d7110dffa8cd7844252025dd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T03:13:43,251 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing b96380a8d7110dffa8cd7844252025dd, disabling compactions & flushes 2024-11-15T03:13:43,251 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ooo,1731640423069.b96380a8d7110dffa8cd7844252025dd. 2024-11-15T03:13:43,251 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1731640423069.b96380a8d7110dffa8cd7844252025dd. 2024-11-15T03:13:43,252 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1731640423069.b96380a8d7110dffa8cd7844252025dd. after waiting 0 ms 2024-11-15T03:13:43,252 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1731640423069.b96380a8d7110dffa8cd7844252025dd. 2024-11-15T03:13:43,252 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ooo,1731640423069.b96380a8d7110dffa8cd7844252025dd. 2024-11-15T03:13:43,252 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for b96380a8d7110dffa8cd7844252025dd: Waiting for close lock at 1731640423251Disabling compacts and flushes for region at 1731640423251Disabling writes for close at 1731640423252 (+1 ms)Writing region close event to WAL at 1731640423252Closed at 1731640423252 2024-11-15T03:13:43,253 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => dc930ecfca79abbf57ed28c20f79230f, NAME => 'test,rrr,1731640423069.dc930ecfca79abbf57ed28c20f79230f.', STARTKEY => 'rrr', ENDKEY => 'sss'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46 2024-11-15T03:13:43,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36559 is added to blk_1073741853_1029 (size=45) 2024-11-15T03:13:43,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38319 is added to blk_1073741853_1029 (size=45) 2024-11-15T03:13:43,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46785 is added to blk_1073741853_1029 (size=45) 2024-11-15T03:13:43,257 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ppp,1731640423069.fac2a1b464a5ff2d581af6baef327bde.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T03:13:43,257 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing fac2a1b464a5ff2d581af6baef327bde, disabling compactions & flushes 2024-11-15T03:13:43,257 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ppp,1731640423069.fac2a1b464a5ff2d581af6baef327bde. 2024-11-15T03:13:43,257 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1731640423069.fac2a1b464a5ff2d581af6baef327bde. 2024-11-15T03:13:43,257 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1731640423069.fac2a1b464a5ff2d581af6baef327bde. after waiting 0 ms 2024-11-15T03:13:43,258 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1731640423069.fac2a1b464a5ff2d581af6baef327bde. 2024-11-15T03:13:43,258 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ppp,1731640423069.fac2a1b464a5ff2d581af6baef327bde. 2024-11-15T03:13:43,258 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for fac2a1b464a5ff2d581af6baef327bde: Waiting for close lock at 1731640423257Disabling compacts and flushes for region at 1731640423257Disabling writes for close at 1731640423257Writing region close event to WAL at 1731640423258 (+1 ms)Closed at 1731640423258 2024-11-15T03:13:43,258 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 4ef03d91ad777a77cf66f510e28419ba, NAME => 'test,sss,1731640423069.4ef03d91ad777a77cf66f510e28419ba.', STARTKEY => 'sss', ENDKEY => 'ttt'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46 2024-11-15T03:13:43,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36559 is added to blk_1073741854_1030 (size=45) 2024-11-15T03:13:43,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46785 is added to blk_1073741854_1030 (size=45) 2024-11-15T03:13:43,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38319 is added to blk_1073741854_1030 (size=45) 2024-11-15T03:13:43,264 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,qqq,1731640423069.23f804d67a47ba185ffd6f6bc7b75464.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T03:13:43,265 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 23f804d67a47ba185ffd6f6bc7b75464, disabling compactions & flushes 2024-11-15T03:13:43,265 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,qqq,1731640423069.23f804d67a47ba185ffd6f6bc7b75464. 2024-11-15T03:13:43,265 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1731640423069.23f804d67a47ba185ffd6f6bc7b75464. 2024-11-15T03:13:43,265 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1731640423069.23f804d67a47ba185ffd6f6bc7b75464. after waiting 0 ms 2024-11-15T03:13:43,265 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1731640423069.23f804d67a47ba185ffd6f6bc7b75464. 2024-11-15T03:13:43,265 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,qqq,1731640423069.23f804d67a47ba185ffd6f6bc7b75464. 2024-11-15T03:13:43,265 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 23f804d67a47ba185ffd6f6bc7b75464: Waiting for close lock at 1731640423265Disabling compacts and flushes for region at 1731640423265Disabling writes for close at 1731640423265Writing region close event to WAL at 1731640423265Closed at 1731640423265 2024-11-15T03:13:43,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38319 is added to blk_1073741855_1031 (size=45) 2024-11-15T03:13:43,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36559 is added to blk_1073741855_1031 (size=45) 2024-11-15T03:13:43,266 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 21df709a703798d68290a0a63322957f, NAME => 'test,ttt,1731640423069.21df709a703798d68290a0a63322957f.', STARTKEY => 'ttt', ENDKEY => 'uuu'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46 2024-11-15T03:13:43,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46785 is added to blk_1073741855_1031 (size=45) 2024-11-15T03:13:43,267 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,rrr,1731640423069.dc930ecfca79abbf57ed28c20f79230f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T03:13:43,267 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing dc930ecfca79abbf57ed28c20f79230f, disabling compactions & flushes 2024-11-15T03:13:43,267 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,rrr,1731640423069.dc930ecfca79abbf57ed28c20f79230f. 2024-11-15T03:13:43,268 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1731640423069.dc930ecfca79abbf57ed28c20f79230f. 2024-11-15T03:13:43,268 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1731640423069.dc930ecfca79abbf57ed28c20f79230f. after waiting 0 ms 2024-11-15T03:13:43,268 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1731640423069.dc930ecfca79abbf57ed28c20f79230f. 2024-11-15T03:13:43,268 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,rrr,1731640423069.dc930ecfca79abbf57ed28c20f79230f. 2024-11-15T03:13:43,268 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for dc930ecfca79abbf57ed28c20f79230f: Waiting for close lock at 1731640423267Disabling compacts and flushes for region at 1731640423267Disabling writes for close at 1731640423268 (+1 ms)Writing region close event to WAL at 1731640423268Closed at 1731640423268 2024-11-15T03:13:43,268 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1b8f1ea215e8c2f78186b80dafed1b28, NAME => 'test,uuu,1731640423069.1b8f1ea215e8c2f78186b80dafed1b28.', STARTKEY => 'uuu', ENDKEY => 'vvv'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46 2024-11-15T03:13:43,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46785 is added to blk_1073741857_1033 (size=45) 2024-11-15T03:13:43,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36559 is added to blk_1073741857_1033 (size=45) 2024-11-15T03:13:43,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46785 is added to blk_1073741858_1034 (size=45) 2024-11-15T03:13:43,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38319 is added to blk_1073741857_1033 (size=45) 2024-11-15T03:13:43,279 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ttt,1731640423069.21df709a703798d68290a0a63322957f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T03:13:43,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38319 is added to blk_1073741858_1034 (size=45) 2024-11-15T03:13:43,280 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 21df709a703798d68290a0a63322957f, disabling compactions & flushes 2024-11-15T03:13:43,280 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ttt,1731640423069.21df709a703798d68290a0a63322957f. 2024-11-15T03:13:43,280 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1731640423069.21df709a703798d68290a0a63322957f. 2024-11-15T03:13:43,280 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1731640423069.21df709a703798d68290a0a63322957f. after waiting 0 ms 2024-11-15T03:13:43,280 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1731640423069.21df709a703798d68290a0a63322957f. 2024-11-15T03:13:43,280 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ttt,1731640423069.21df709a703798d68290a0a63322957f. 2024-11-15T03:13:43,280 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 21df709a703798d68290a0a63322957f: Waiting for close lock at 1731640423280Disabling compacts and flushes for region at 1731640423280Disabling writes for close at 1731640423280Writing region close event to WAL at 1731640423280Closed at 1731640423280 2024-11-15T03:13:43,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36559 is added to blk_1073741858_1034 (size=45) 2024-11-15T03:13:43,281 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => dc93af816bdf94cf920a5adc31356783, NAME => 'test,vvv,1731640423069.dc93af816bdf94cf920a5adc31356783.', STARTKEY => 'vvv', ENDKEY => 'www'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46 2024-11-15T03:13:43,281 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,uuu,1731640423069.1b8f1ea215e8c2f78186b80dafed1b28.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T03:13:43,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46785 is added to blk_1073741856_1032 (size=45) 2024-11-15T03:13:43,281 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 1b8f1ea215e8c2f78186b80dafed1b28, disabling compactions & flushes 2024-11-15T03:13:43,281 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,uuu,1731640423069.1b8f1ea215e8c2f78186b80dafed1b28. 2024-11-15T03:13:43,281 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1731640423069.1b8f1ea215e8c2f78186b80dafed1b28. 2024-11-15T03:13:43,282 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1731640423069.1b8f1ea215e8c2f78186b80dafed1b28. after waiting 0 ms 2024-11-15T03:13:43,282 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1731640423069.1b8f1ea215e8c2f78186b80dafed1b28. 2024-11-15T03:13:43,282 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,uuu,1731640423069.1b8f1ea215e8c2f78186b80dafed1b28. 2024-11-15T03:13:43,282 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 1b8f1ea215e8c2f78186b80dafed1b28: Waiting for close lock at 1731640423281Disabling compacts and flushes for region at 1731640423281Disabling writes for close at 1731640423282 (+1 ms)Writing region close event to WAL at 1731640423282Closed at 1731640423282 2024-11-15T03:13:43,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38319 is added to blk_1073741856_1032 (size=45) 2024-11-15T03:13:43,283 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 155ef591549501ad2da24d2c559fb7f9, NAME => 'test,www,1731640423069.155ef591549501ad2da24d2c559fb7f9.', STARTKEY => 'www', ENDKEY => 'xxx'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46 2024-11-15T03:13:43,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36559 is added to blk_1073741856_1032 (size=45) 2024-11-15T03:13:43,290 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,sss,1731640423069.4ef03d91ad777a77cf66f510e28419ba.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T03:13:43,290 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 4ef03d91ad777a77cf66f510e28419ba, disabling compactions & flushes 2024-11-15T03:13:43,290 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,sss,1731640423069.4ef03d91ad777a77cf66f510e28419ba. 2024-11-15T03:13:43,290 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1731640423069.4ef03d91ad777a77cf66f510e28419ba. 2024-11-15T03:13:43,290 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,sss,1731640423069.4ef03d91ad777a77cf66f510e28419ba. after waiting 0 ms 2024-11-15T03:13:43,290 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,sss,1731640423069.4ef03d91ad777a77cf66f510e28419ba. 2024-11-15T03:13:43,290 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,sss,1731640423069.4ef03d91ad777a77cf66f510e28419ba. 2024-11-15T03:13:43,290 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 4ef03d91ad777a77cf66f510e28419ba: Waiting for close lock at 1731640423290Disabling compacts and flushes for region at 1731640423290Disabling writes for close at 1731640423290Writing region close event to WAL at 1731640423290Closed at 1731640423290 2024-11-15T03:13:43,291 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 6cca2d3f6a001d96f34b7a93afec8fe7, NAME => 'test,xxx,1731640423069.6cca2d3f6a001d96f34b7a93afec8fe7.', STARTKEY => 'xxx', ENDKEY => 'yyy'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46 2024-11-15T03:13:43,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36559 is added to blk_1073741859_1035 (size=45) 2024-11-15T03:13:43,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46785 is added to blk_1073741860_1036 (size=45) 2024-11-15T03:13:43,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38319 is added to blk_1073741860_1036 (size=45) 2024-11-15T03:13:43,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36559 is added to blk_1073741860_1036 (size=45) 2024-11-15T03:13:43,302 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,www,1731640423069.155ef591549501ad2da24d2c559fb7f9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T03:13:43,302 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 155ef591549501ad2da24d2c559fb7f9, disabling compactions & flushes 2024-11-15T03:13:43,302 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,www,1731640423069.155ef591549501ad2da24d2c559fb7f9. 2024-11-15T03:13:43,302 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1731640423069.155ef591549501ad2da24d2c559fb7f9. 2024-11-15T03:13:43,302 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,www,1731640423069.155ef591549501ad2da24d2c559fb7f9. after waiting 0 ms 2024-11-15T03:13:43,302 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,www,1731640423069.155ef591549501ad2da24d2c559fb7f9. 2024-11-15T03:13:43,302 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,www,1731640423069.155ef591549501ad2da24d2c559fb7f9. 2024-11-15T03:13:43,303 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 155ef591549501ad2da24d2c559fb7f9: Waiting for close lock at 1731640423302Disabling compacts and flushes for region at 1731640423302Disabling writes for close at 1731640423302Writing region close event to WAL at 1731640423302Closed at 1731640423302 2024-11-15T03:13:43,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46785 is added to blk_1073741859_1035 (size=45) 2024-11-15T03:13:43,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38319 is added to blk_1073741859_1035 (size=45) 2024-11-15T03:13:43,303 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => e0bce1a97a9c7f320b315db443b47c23, NAME => 'test,yyy,1731640423069.e0bce1a97a9c7f320b315db443b47c23.', STARTKEY => 'yyy', ENDKEY => 'zzz'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46 2024-11-15T03:13:43,304 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,vvv,1731640423069.dc93af816bdf94cf920a5adc31356783.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T03:13:43,304 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing dc93af816bdf94cf920a5adc31356783, disabling compactions & flushes 2024-11-15T03:13:43,304 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,vvv,1731640423069.dc93af816bdf94cf920a5adc31356783. 2024-11-15T03:13:43,304 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1731640423069.dc93af816bdf94cf920a5adc31356783. 2024-11-15T03:13:43,304 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1731640423069.dc93af816bdf94cf920a5adc31356783. after waiting 0 ms 2024-11-15T03:13:43,305 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1731640423069.dc93af816bdf94cf920a5adc31356783. 2024-11-15T03:13:43,305 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,vvv,1731640423069.dc93af816bdf94cf920a5adc31356783. 2024-11-15T03:13:43,305 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for dc93af816bdf94cf920a5adc31356783: Waiting for close lock at 1731640423304Disabling compacts and flushes for region at 1731640423304Disabling writes for close at 1731640423304Writing region close event to WAL at 1731640423305 (+1 ms)Closed at 1731640423305 2024-11-15T03:13:43,305 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 17e8b3b889a606b8c6826552e784d401, NAME => 'test,zzz,1731640423069.17e8b3b889a606b8c6826552e784d401.', STARTKEY => 'zzz', ENDKEY => ''}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46 2024-11-15T03:13:43,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46785 is added to blk_1073741861_1037 (size=45) 2024-11-15T03:13:43,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36559 is added to blk_1073741861_1037 (size=45) 2024-11-15T03:13:43,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38319 is added to blk_1073741861_1037 (size=45) 2024-11-15T03:13:43,313 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,xxx,1731640423069.6cca2d3f6a001d96f34b7a93afec8fe7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T03:13:43,313 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 6cca2d3f6a001d96f34b7a93afec8fe7, disabling compactions & flushes 2024-11-15T03:13:43,313 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,xxx,1731640423069.6cca2d3f6a001d96f34b7a93afec8fe7. 2024-11-15T03:13:43,313 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1731640423069.6cca2d3f6a001d96f34b7a93afec8fe7. 2024-11-15T03:13:43,313 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1731640423069.6cca2d3f6a001d96f34b7a93afec8fe7. after waiting 0 ms 2024-11-15T03:13:43,313 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1731640423069.6cca2d3f6a001d96f34b7a93afec8fe7. 2024-11-15T03:13:43,313 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,xxx,1731640423069.6cca2d3f6a001d96f34b7a93afec8fe7. 2024-11-15T03:13:43,314 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 6cca2d3f6a001d96f34b7a93afec8fe7: Waiting for close lock at 1731640423313Disabling compacts and flushes for region at 1731640423313Disabling writes for close at 1731640423313Writing region close event to WAL at 1731640423313Closed at 1731640423313 2024-11-15T03:13:43,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36559 is added to blk_1073741862_1038 (size=45) 2024-11-15T03:13:43,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38319 is added to blk_1073741862_1038 (size=45) 2024-11-15T03:13:43,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46785 is added to blk_1073741862_1038 (size=45) 2024-11-15T03:13:43,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46785 is added to blk_1073741863_1039 (size=42) 2024-11-15T03:13:43,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38319 is added to blk_1073741863_1039 (size=42) 2024-11-15T03:13:43,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36559 is added to blk_1073741863_1039 (size=42) 2024-11-15T03:13:43,331 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,zzz,1731640423069.17e8b3b889a606b8c6826552e784d401.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T03:13:43,331 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 17e8b3b889a606b8c6826552e784d401, disabling compactions & flushes 2024-11-15T03:13:43,331 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,zzz,1731640423069.17e8b3b889a606b8c6826552e784d401. 2024-11-15T03:13:43,331 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1731640423069.17e8b3b889a606b8c6826552e784d401. 2024-11-15T03:13:43,331 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1731640423069.17e8b3b889a606b8c6826552e784d401. after waiting 0 ms 2024-11-15T03:13:43,331 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1731640423069.17e8b3b889a606b8c6826552e784d401. 2024-11-15T03:13:43,332 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,zzz,1731640423069.17e8b3b889a606b8c6826552e784d401. 2024-11-15T03:13:43,332 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 17e8b3b889a606b8c6826552e784d401: Waiting for close lock at 1731640423331Disabling compacts and flushes for region at 1731640423331Disabling writes for close at 1731640423331Writing region close event to WAL at 1731640423331Closed at 1731640423331 2024-11-15T03:13:43,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T03:13:43,721 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,yyy,1731640423069.e0bce1a97a9c7f320b315db443b47c23.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T03:13:43,721 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing e0bce1a97a9c7f320b315db443b47c23, disabling compactions & flushes 2024-11-15T03:13:43,721 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,yyy,1731640423069.e0bce1a97a9c7f320b315db443b47c23. 2024-11-15T03:13:43,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T03:13:43,721 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1731640423069.e0bce1a97a9c7f320b315db443b47c23. 2024-11-15T03:13:43,721 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1731640423069.e0bce1a97a9c7f320b315db443b47c23. after waiting 0 ms 2024-11-15T03:13:43,722 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1731640423069.e0bce1a97a9c7f320b315db443b47c23. 2024-11-15T03:13:43,722 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,yyy,1731640423069.e0bce1a97a9c7f320b315db443b47c23. 2024-11-15T03:13:43,722 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for e0bce1a97a9c7f320b315db443b47c23: Waiting for close lock at 1731640423721Disabling compacts and flushes for region at 1731640423721Disabling writes for close at 1731640423721Writing region close event to WAL at 1731640423722 (+1 ms)Closed at 1731640423722 2024-11-15T03:13:43,727 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T03:13:43,735 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,bbb,1731640423069.6130297c45bd23340b368de2cc708c90.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731640423727"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731640423727"}]},"ts":"1731640423727"} 2024-11-15T03:13:43,735 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ccc,1731640423069.5971f307349447e7f24b0dd05b96cffc.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731640423727"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731640423727"}]},"ts":"1731640423727"} 2024-11-15T03:13:43,736 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,,1731640423069.5965b596ce7b87fabde47569d60c3f95.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1731640423727"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731640423727"}]},"ts":"1731640423727"} 2024-11-15T03:13:43,736 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,fff,1731640423069.c4b611da4a73d4cfc99379713486e21c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731640423727"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731640423727"}]},"ts":"1731640423727"} 2024-11-15T03:13:43,736 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ddd,1731640423069.1beb579fcf8e161a0b9c2051c671a8f2.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731640423727"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731640423727"}]},"ts":"1731640423727"} 2024-11-15T03:13:43,736 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,eee,1731640423069.e0f9e1e28ca8906f0f82f959a60c55ce.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731640423727"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731640423727"}]},"ts":"1731640423727"} 2024-11-15T03:13:43,736 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,hhh,1731640423069.48ff514f5498660a4e8fc753340bb847.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731640423727"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731640423727"}]},"ts":"1731640423727"} 2024-11-15T03:13:43,737 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ggg,1731640423069.d0eff1b91bc9b77579a5c44f1c985985.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731640423727"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731640423727"}]},"ts":"1731640423727"} 2024-11-15T03:13:43,737 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,iii,1731640423069.f5219cfe492c74eb8312930d328964b6.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731640423727"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731640423727"}]},"ts":"1731640423727"} 2024-11-15T03:13:43,737 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,jjj,1731640423069.b1704275babe02e36b29cfaadf5b81af.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731640423727"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731640423727"}]},"ts":"1731640423727"} 2024-11-15T03:13:43,737 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,kkk,1731640423069.bb2ead3b463edc0f2b603da7d19f1201.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731640423727"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731640423727"}]},"ts":"1731640423727"} 2024-11-15T03:13:43,737 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,lll,1731640423069.860df8edb65f2fdd3c8bc00f89f94ba0.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731640423727"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731640423727"}]},"ts":"1731640423727"} 2024-11-15T03:13:43,738 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,mmm,1731640423069.eaf85cc654192e46d3ea0904493eea12.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731640423727"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731640423727"}]},"ts":"1731640423727"} 2024-11-15T03:13:43,738 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,nnn,1731640423069.a73d2d096e2f14a7e9365ac762927d55.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731640423727"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731640423727"}]},"ts":"1731640423727"} 2024-11-15T03:13:43,738 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ooo,1731640423069.b96380a8d7110dffa8cd7844252025dd.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731640423727"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731640423727"}]},"ts":"1731640423727"} 2024-11-15T03:13:43,738 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ppp,1731640423069.fac2a1b464a5ff2d581af6baef327bde.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731640423727"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731640423727"}]},"ts":"1731640423727"} 2024-11-15T03:13:43,738 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,qqq,1731640423069.23f804d67a47ba185ffd6f6bc7b75464.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731640423727"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731640423727"}]},"ts":"1731640423727"} 2024-11-15T03:13:43,739 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,rrr,1731640423069.dc930ecfca79abbf57ed28c20f79230f.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731640423727"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731640423727"}]},"ts":"1731640423727"} 2024-11-15T03:13:43,739 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ttt,1731640423069.21df709a703798d68290a0a63322957f.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731640423727"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731640423727"}]},"ts":"1731640423727"} 2024-11-15T03:13:43,739 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,uuu,1731640423069.1b8f1ea215e8c2f78186b80dafed1b28.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731640423727"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731640423727"}]},"ts":"1731640423727"} 2024-11-15T03:13:43,739 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,sss,1731640423069.4ef03d91ad777a77cf66f510e28419ba.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731640423727"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731640423727"}]},"ts":"1731640423727"} 2024-11-15T03:13:43,739 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,www,1731640423069.155ef591549501ad2da24d2c559fb7f9.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731640423727"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731640423727"}]},"ts":"1731640423727"} 2024-11-15T03:13:43,739 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,vvv,1731640423069.dc93af816bdf94cf920a5adc31356783.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731640423727"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731640423727"}]},"ts":"1731640423727"} 2024-11-15T03:13:43,740 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,xxx,1731640423069.6cca2d3f6a001d96f34b7a93afec8fe7.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731640423727"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731640423727"}]},"ts":"1731640423727"} 2024-11-15T03:13:43,740 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,zzz,1731640423069.17e8b3b889a606b8c6826552e784d401.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1731640423727"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731640423727"}]},"ts":"1731640423727"} 2024-11-15T03:13:43,740 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,yyy,1731640423069.e0bce1a97a9c7f320b315db443b47c23.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731640423727"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731640423727"}]},"ts":"1731640423727"} 2024-11-15T03:13:43,779 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 26 regions to meta. 2024-11-15T03:13:43,781 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T03:13:43,784 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731640423781"}]},"ts":"1731640423781"} 2024-11-15T03:13:43,788 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLING in hbase:meta 2024-11-15T03:13:43,789 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {91f82191efb8=0} racks are {/default-rack=0} 2024-11-15T03:13:43,808 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-15T03:13:43,808 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-15T03:13:43,808 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-15T03:13:43,808 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-15T03:13:43,808 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-15T03:13:43,808 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-15T03:13:43,808 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-15T03:13:43,808 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-15T03:13:43,808 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-15T03:13:43,808 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-15T03:13:43,811 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5965b596ce7b87fabde47569d60c3f95, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6130297c45bd23340b368de2cc708c90, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1beb579fcf8e161a0b9c2051c671a8f2, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e0f9e1e28ca8906f0f82f959a60c55ce, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c4b611da4a73d4cfc99379713486e21c, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d0eff1b91bc9b77579a5c44f1c985985, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=48ff514f5498660a4e8fc753340bb847, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f5219cfe492c74eb8312930d328964b6, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b1704275babe02e36b29cfaadf5b81af, ASSIGN}, {pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=bb2ead3b463edc0f2b603da7d19f1201, ASSIGN}, {pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=860df8edb65f2fdd3c8bc00f89f94ba0, ASSIGN}, {pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=eaf85cc654192e46d3ea0904493eea12, ASSIGN}, {pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN}, {pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b96380a8d7110dffa8cd7844252025dd, ASSIGN}, {pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fac2a1b464a5ff2d581af6baef327bde, ASSIGN}, {pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=23f804d67a47ba185ffd6f6bc7b75464, ASSIGN}, {pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=dc930ecfca79abbf57ed28c20f79230f, ASSIGN}, {pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4ef03d91ad777a77cf66f510e28419ba, ASSIGN}, {pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=21df709a703798d68290a0a63322957f, ASSIGN}, {pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1b8f1ea215e8c2f78186b80dafed1b28, ASSIGN}, {pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=dc93af816bdf94cf920a5adc31356783, ASSIGN}, {pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=155ef591549501ad2da24d2c559fb7f9, ASSIGN}, {pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6cca2d3f6a001d96f34b7a93afec8fe7, ASSIGN}, {pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN}, {pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=17e8b3b889a606b8c6826552e784d401, ASSIGN}] 2024-11-15T03:13:43,819 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=17e8b3b889a606b8c6826552e784d401, ASSIGN 2024-11-15T03:13:43,820 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=17e8b3b889a606b8c6826552e784d401, ASSIGN; state=OFFLINE, location=91f82191efb8,37523,1731640420593; forceNewPlan=false, retain=false 2024-11-15T03:13:43,821 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN 2024-11-15T03:13:43,822 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN; state=OFFLINE, location=91f82191efb8,42027,1731640420387; forceNewPlan=false, retain=false 2024-11-15T03:13:43,824 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6cca2d3f6a001d96f34b7a93afec8fe7, ASSIGN 2024-11-15T03:13:43,825 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=6cca2d3f6a001d96f34b7a93afec8fe7, ASSIGN; state=OFFLINE, location=91f82191efb8,41451,1731640420638; forceNewPlan=false, retain=false 2024-11-15T03:13:43,827 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=155ef591549501ad2da24d2c559fb7f9, ASSIGN 2024-11-15T03:13:43,828 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=155ef591549501ad2da24d2c559fb7f9, ASSIGN; state=OFFLINE, location=91f82191efb8,37523,1731640420593; forceNewPlan=false, retain=false 2024-11-15T03:13:43,830 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=dc93af816bdf94cf920a5adc31356783, ASSIGN 2024-11-15T03:13:43,831 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=dc93af816bdf94cf920a5adc31356783, ASSIGN; state=OFFLINE, location=91f82191efb8,42027,1731640420387; forceNewPlan=false, retain=false 2024-11-15T03:13:43,833 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1b8f1ea215e8c2f78186b80dafed1b28, ASSIGN 2024-11-15T03:13:43,834 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=1b8f1ea215e8c2f78186b80dafed1b28, ASSIGN; state=OFFLINE, location=91f82191efb8,42027,1731640420387; forceNewPlan=false, retain=false 2024-11-15T03:13:43,835 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=21df709a703798d68290a0a63322957f, ASSIGN 2024-11-15T03:13:43,836 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=21df709a703798d68290a0a63322957f, ASSIGN; state=OFFLINE, location=91f82191efb8,37523,1731640420593; forceNewPlan=false, retain=false 2024-11-15T03:13:43,838 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4ef03d91ad777a77cf66f510e28419ba, ASSIGN 2024-11-15T03:13:43,839 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4ef03d91ad777a77cf66f510e28419ba, ASSIGN; state=OFFLINE, location=91f82191efb8,41451,1731640420638; forceNewPlan=false, retain=false 2024-11-15T03:13:43,840 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=dc930ecfca79abbf57ed28c20f79230f, ASSIGN 2024-11-15T03:13:43,842 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=dc930ecfca79abbf57ed28c20f79230f, ASSIGN; state=OFFLINE, location=91f82191efb8,41451,1731640420638; forceNewPlan=false, retain=false 2024-11-15T03:13:43,843 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=23f804d67a47ba185ffd6f6bc7b75464, ASSIGN 2024-11-15T03:13:43,844 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=23f804d67a47ba185ffd6f6bc7b75464, ASSIGN; state=OFFLINE, location=91f82191efb8,42027,1731640420387; forceNewPlan=false, retain=false 2024-11-15T03:13:43,846 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fac2a1b464a5ff2d581af6baef327bde, ASSIGN 2024-11-15T03:13:43,847 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=fac2a1b464a5ff2d581af6baef327bde, ASSIGN; state=OFFLINE, location=91f82191efb8,37523,1731640420593; forceNewPlan=false, retain=false 2024-11-15T03:13:43,849 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b96380a8d7110dffa8cd7844252025dd, ASSIGN 2024-11-15T03:13:43,850 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=b96380a8d7110dffa8cd7844252025dd, ASSIGN; state=OFFLINE, location=91f82191efb8,41451,1731640420638; forceNewPlan=false, retain=false 2024-11-15T03:13:43,851 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN 2024-11-15T03:13:43,853 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN; state=OFFLINE, location=91f82191efb8,42027,1731640420387; forceNewPlan=false, retain=false 2024-11-15T03:13:43,854 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=eaf85cc654192e46d3ea0904493eea12, ASSIGN 2024-11-15T03:13:43,855 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=eaf85cc654192e46d3ea0904493eea12, ASSIGN; state=OFFLINE, location=91f82191efb8,37523,1731640420593; forceNewPlan=false, retain=false 2024-11-15T03:13:43,857 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=860df8edb65f2fdd3c8bc00f89f94ba0, ASSIGN 2024-11-15T03:13:43,859 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=860df8edb65f2fdd3c8bc00f89f94ba0, ASSIGN; state=OFFLINE, location=91f82191efb8,41451,1731640420638; forceNewPlan=false, retain=false 2024-11-15T03:13:43,860 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=bb2ead3b463edc0f2b603da7d19f1201, ASSIGN 2024-11-15T03:13:43,862 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=bb2ead3b463edc0f2b603da7d19f1201, ASSIGN; state=OFFLINE, location=91f82191efb8,42027,1731640420387; forceNewPlan=false, retain=false 2024-11-15T03:13:43,863 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b1704275babe02e36b29cfaadf5b81af, ASSIGN 2024-11-15T03:13:43,865 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=b1704275babe02e36b29cfaadf5b81af, ASSIGN; state=OFFLINE, location=91f82191efb8,37523,1731640420593; forceNewPlan=false, retain=false 2024-11-15T03:13:43,867 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f5219cfe492c74eb8312930d328964b6, ASSIGN 2024-11-15T03:13:43,868 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f5219cfe492c74eb8312930d328964b6, ASSIGN; state=OFFLINE, location=91f82191efb8,41451,1731640420638; forceNewPlan=false, retain=false 2024-11-15T03:13:43,870 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=48ff514f5498660a4e8fc753340bb847, ASSIGN 2024-11-15T03:13:43,871 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=48ff514f5498660a4e8fc753340bb847, ASSIGN; state=OFFLINE, location=91f82191efb8,37523,1731640420593; forceNewPlan=false, retain=false 2024-11-15T03:13:43,873 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d0eff1b91bc9b77579a5c44f1c985985, ASSIGN 2024-11-15T03:13:43,874 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d0eff1b91bc9b77579a5c44f1c985985, ASSIGN; state=OFFLINE, location=91f82191efb8,42027,1731640420387; forceNewPlan=false, retain=false 2024-11-15T03:13:43,876 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c4b611da4a73d4cfc99379713486e21c, ASSIGN 2024-11-15T03:13:43,877 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c4b611da4a73d4cfc99379713486e21c, ASSIGN; state=OFFLINE, location=91f82191efb8,37523,1731640420593; forceNewPlan=false, retain=false 2024-11-15T03:13:43,879 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e0f9e1e28ca8906f0f82f959a60c55ce, ASSIGN 2024-11-15T03:13:43,880 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=e0f9e1e28ca8906f0f82f959a60c55ce, ASSIGN; state=OFFLINE, location=91f82191efb8,41451,1731640420638; forceNewPlan=false, retain=false 2024-11-15T03:13:43,882 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1beb579fcf8e161a0b9c2051c671a8f2, ASSIGN 2024-11-15T03:13:43,884 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=1beb579fcf8e161a0b9c2051c671a8f2, ASSIGN; state=OFFLINE, location=91f82191efb8,42027,1731640420387; forceNewPlan=false, retain=false 2024-11-15T03:13:43,885 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN 2024-11-15T03:13:43,887 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN; state=OFFLINE, location=91f82191efb8,42027,1731640420387; forceNewPlan=false, retain=false 2024-11-15T03:13:43,889 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6130297c45bd23340b368de2cc708c90, ASSIGN 2024-11-15T03:13:43,890 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=6130297c45bd23340b368de2cc708c90, ASSIGN; state=OFFLINE, location=91f82191efb8,37523,1731640420593; forceNewPlan=false, retain=false 2024-11-15T03:13:43,892 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5965b596ce7b87fabde47569d60c3f95, ASSIGN 2024-11-15T03:13:43,894 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=5965b596ce7b87fabde47569d60c3f95, ASSIGN; state=OFFLINE, location=91f82191efb8,41451,1731640420638; forceNewPlan=false, retain=false 2024-11-15T03:13:43,976 INFO [91f82191efb8:42051 {}] balancer.BaseLoadBalancer(388): Reassigned 26 regions. 26 retained the pre-restart assignment. 2024-11-15T03:13:43,977 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=e0bce1a97a9c7f320b315db443b47c23, regionState=OPENING, regionLocation=91f82191efb8,42027,1731640420387 2024-11-15T03:13:43,979 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=dc93af816bdf94cf920a5adc31356783, regionState=OPENING, regionLocation=91f82191efb8,42027,1731640420387 2024-11-15T03:13:43,980 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=1b8f1ea215e8c2f78186b80dafed1b28, regionState=OPENING, regionLocation=91f82191efb8,42027,1731640420387 2024-11-15T03:13:43,981 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=23f804d67a47ba185ffd6f6bc7b75464, regionState=OPENING, regionLocation=91f82191efb8,42027,1731640420387 2024-11-15T03:13:43,982 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=a73d2d096e2f14a7e9365ac762927d55, regionState=OPENING, regionLocation=91f82191efb8,42027,1731640420387 2024-11-15T03:13:43,982 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN because future has completed 2024-11-15T03:13:43,983 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=bb2ead3b463edc0f2b603da7d19f1201, regionState=OPENING, regionLocation=91f82191efb8,42027,1731640420387 2024-11-15T03:13:43,984 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure e0bce1a97a9c7f320b315db443b47c23, server=91f82191efb8,42027,1731640420387}] 2024-11-15T03:13:43,985 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=dc93af816bdf94cf920a5adc31356783, ASSIGN because future has completed 2024-11-15T03:13:43,988 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=1b8f1ea215e8c2f78186b80dafed1b28, ASSIGN because future has completed 2024-11-15T03:13:43,992 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=23f804d67a47ba185ffd6f6bc7b75464, ASSIGN because future has completed 2024-11-15T03:13:43,995 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 23f804d67a47ba185ffd6f6bc7b75464, server=91f82191efb8,42027,1731640420387}] 2024-11-15T03:13:43,995 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN because future has completed 2024-11-15T03:13:43,998 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=bb2ead3b463edc0f2b603da7d19f1201, ASSIGN because future has completed 2024-11-15T03:13:44,004 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,42027,1731640420387}] 2024-11-15T03:13:44,011 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure a73d2d096e2f14a7e9365ac762927d55, server=91f82191efb8,42027,1731640420387}] 2024-11-15T03:13:44,018 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1b8f1ea215e8c2f78186b80dafed1b28, server=91f82191efb8,42027,1731640420387}] 2024-11-15T03:13:44,022 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure dc93af816bdf94cf920a5adc31356783, server=91f82191efb8,42027,1731640420387}] 2024-11-15T03:13:44,027 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=d0eff1b91bc9b77579a5c44f1c985985, regionState=OPENING, regionLocation=91f82191efb8,42027,1731640420387 2024-11-15T03:13:44,028 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1beb579fcf8e161a0b9c2051c671a8f2, regionState=OPENING, regionLocation=91f82191efb8,42027,1731640420387 2024-11-15T03:13:44,029 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=5971f307349447e7f24b0dd05b96cffc, regionState=OPENING, regionLocation=91f82191efb8,42027,1731640420387 2024-11-15T03:13:44,029 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=6cca2d3f6a001d96f34b7a93afec8fe7, regionState=OPENING, regionLocation=91f82191efb8,41451,1731640420638 2024-11-15T03:13:44,030 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=4ef03d91ad777a77cf66f510e28419ba, regionState=OPENING, regionLocation=91f82191efb8,41451,1731640420638 2024-11-15T03:13:44,031 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d0eff1b91bc9b77579a5c44f1c985985, ASSIGN because future has completed 2024-11-15T03:13:44,031 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=dc930ecfca79abbf57ed28c20f79230f, regionState=OPENING, regionLocation=91f82191efb8,41451,1731640420638 2024-11-15T03:13:44,032 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=b96380a8d7110dffa8cd7844252025dd, regionState=OPENING, regionLocation=91f82191efb8,41451,1731640420638 2024-11-15T03:13:44,032 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure d0eff1b91bc9b77579a5c44f1c985985, server=91f82191efb8,42027,1731640420387}] 2024-11-15T03:13:44,033 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=1beb579fcf8e161a0b9c2051c671a8f2, ASSIGN because future has completed 2024-11-15T03:13:44,036 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN because future has completed 2024-11-15T03:13:44,040 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5971f307349447e7f24b0dd05b96cffc, server=91f82191efb8,42027,1731640420387}] 2024-11-15T03:13:44,041 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=6cca2d3f6a001d96f34b7a93afec8fe7, ASSIGN because future has completed 2024-11-15T03:13:44,043 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4ef03d91ad777a77cf66f510e28419ba, ASSIGN because future has completed 2024-11-15T03:13:44,045 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=dc930ecfca79abbf57ed28c20f79230f, ASSIGN because future has completed 2024-11-15T03:13:44,046 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure dc930ecfca79abbf57ed28c20f79230f, server=91f82191efb8,41451,1731640420638}] 2024-11-15T03:13:44,047 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=b96380a8d7110dffa8cd7844252025dd, ASSIGN because future has completed 2024-11-15T03:13:44,052 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure b96380a8d7110dffa8cd7844252025dd, server=91f82191efb8,41451,1731640420638}] 2024-11-15T03:13:44,057 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4ef03d91ad777a77cf66f510e28419ba, server=91f82191efb8,41451,1731640420638}] 2024-11-15T03:13:44,062 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6cca2d3f6a001d96f34b7a93afec8fe7, server=91f82191efb8,41451,1731640420638}] 2024-11-15T03:13:44,066 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1beb579fcf8e161a0b9c2051c671a8f2, server=91f82191efb8,42027,1731640420387}] 2024-11-15T03:13:44,071 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=860df8edb65f2fdd3c8bc00f89f94ba0, regionState=OPENING, regionLocation=91f82191efb8,41451,1731640420638 2024-11-15T03:13:44,072 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=f5219cfe492c74eb8312930d328964b6, regionState=OPENING, regionLocation=91f82191efb8,41451,1731640420638 2024-11-15T03:13:44,072 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=e0f9e1e28ca8906f0f82f959a60c55ce, regionState=OPENING, regionLocation=91f82191efb8,41451,1731640420638 2024-11-15T03:13:44,073 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=5965b596ce7b87fabde47569d60c3f95, regionState=OPENING, regionLocation=91f82191efb8,41451,1731640420638 2024-11-15T03:13:44,074 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=860df8edb65f2fdd3c8bc00f89f94ba0, ASSIGN because future has completed 2024-11-15T03:13:44,074 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=17e8b3b889a606b8c6826552e784d401, regionState=OPENING, regionLocation=91f82191efb8,37523,1731640420593 2024-11-15T03:13:44,075 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 860df8edb65f2fdd3c8bc00f89f94ba0, server=91f82191efb8,41451,1731640420638}] 2024-11-15T03:13:44,075 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f5219cfe492c74eb8312930d328964b6, ASSIGN because future has completed 2024-11-15T03:13:44,077 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=e0f9e1e28ca8906f0f82f959a60c55ce, ASSIGN because future has completed 2024-11-15T03:13:44,079 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=5965b596ce7b87fabde47569d60c3f95, ASSIGN because future has completed 2024-11-15T03:13:44,080 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=17e8b3b889a606b8c6826552e784d401, ASSIGN because future has completed 2024-11-15T03:13:44,081 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 17e8b3b889a606b8c6826552e784d401, server=91f82191efb8,37523,1731640420593}] 2024-11-15T03:13:44,086 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5965b596ce7b87fabde47569d60c3f95, server=91f82191efb8,41451,1731640420638}] 2024-11-15T03:13:44,092 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure e0f9e1e28ca8906f0f82f959a60c55ce, server=91f82191efb8,41451,1731640420638}] 2024-11-15T03:13:44,097 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5219cfe492c74eb8312930d328964b6, server=91f82191efb8,41451,1731640420638}] 2024-11-15T03:13:44,102 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=155ef591549501ad2da24d2c559fb7f9, regionState=OPENING, regionLocation=91f82191efb8,37523,1731640420593 2024-11-15T03:13:44,103 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=21df709a703798d68290a0a63322957f, regionState=OPENING, regionLocation=91f82191efb8,37523,1731640420593 2024-11-15T03:13:44,103 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=fac2a1b464a5ff2d581af6baef327bde, regionState=OPENING, regionLocation=91f82191efb8,37523,1731640420593 2024-11-15T03:13:44,104 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=eaf85cc654192e46d3ea0904493eea12, regionState=OPENING, regionLocation=91f82191efb8,37523,1731640420593 2024-11-15T03:13:44,104 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=b1704275babe02e36b29cfaadf5b81af, regionState=OPENING, regionLocation=91f82191efb8,37523,1731640420593 2024-11-15T03:13:44,105 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=48ff514f5498660a4e8fc753340bb847, regionState=OPENING, regionLocation=91f82191efb8,37523,1731640420593 2024-11-15T03:13:44,105 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=155ef591549501ad2da24d2c559fb7f9, ASSIGN because future has completed 2024-11-15T03:13:44,106 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 155ef591549501ad2da24d2c559fb7f9, server=91f82191efb8,37523,1731640420593}] 2024-11-15T03:13:44,107 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=21df709a703798d68290a0a63322957f, ASSIGN because future has completed 2024-11-15T03:13:44,108 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=fac2a1b464a5ff2d581af6baef327bde, ASSIGN because future has completed 2024-11-15T03:13:44,111 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=eaf85cc654192e46d3ea0904493eea12, ASSIGN because future has completed 2024-11-15T03:13:44,112 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=b1704275babe02e36b29cfaadf5b81af, ASSIGN because future has completed 2024-11-15T03:13:44,114 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure b1704275babe02e36b29cfaadf5b81af, server=91f82191efb8,37523,1731640420593}] 2024-11-15T03:13:44,114 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=48ff514f5498660a4e8fc753340bb847, ASSIGN because future has completed 2024-11-15T03:13:44,119 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 48ff514f5498660a4e8fc753340bb847, server=91f82191efb8,37523,1731640420593}] 2024-11-15T03:13:44,122 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure eaf85cc654192e46d3ea0904493eea12, server=91f82191efb8,37523,1731640420593}] 2024-11-15T03:13:44,126 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure fac2a1b464a5ff2d581af6baef327bde, server=91f82191efb8,37523,1731640420593}] 2024-11-15T03:13:44,130 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 21df709a703798d68290a0a63322957f, server=91f82191efb8,37523,1731640420593}] 2024-11-15T03:13:44,135 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=c4b611da4a73d4cfc99379713486e21c, regionState=OPENING, regionLocation=91f82191efb8,37523,1731640420593 2024-11-15T03:13:44,135 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=6130297c45bd23340b368de2cc708c90, regionState=OPENING, regionLocation=91f82191efb8,37523,1731640420593 2024-11-15T03:13:44,138 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c4b611da4a73d4cfc99379713486e21c, ASSIGN because future has completed 2024-11-15T03:13:44,138 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure c4b611da4a73d4cfc99379713486e21c, server=91f82191efb8,37523,1731640420593}] 2024-11-15T03:13:44,139 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=6130297c45bd23340b368de2cc708c90, ASSIGN because future has completed 2024-11-15T03:13:44,143 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6130297c45bd23340b368de2cc708c90, server=91f82191efb8,37523,1731640420593}] 2024-11-15T03:13:44,147 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open test,uuu,1731640423069.1b8f1ea215e8c2f78186b80dafed1b28. 2024-11-15T03:13:44,148 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => 1b8f1ea215e8c2f78186b80dafed1b28, NAME => 'test,uuu,1731640423069.1b8f1ea215e8c2f78186b80dafed1b28.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-15T03:13:44,149 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 1b8f1ea215e8c2f78186b80dafed1b28 2024-11-15T03:13:44,149 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated test,uuu,1731640423069.1b8f1ea215e8c2f78186b80dafed1b28.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T03:13:44,149 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for 1b8f1ea215e8c2f78186b80dafed1b28 2024-11-15T03:13:44,149 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for 1b8f1ea215e8c2f78186b80dafed1b28 2024-11-15T03:13:44,151 INFO [StoreOpener-1b8f1ea215e8c2f78186b80dafed1b28-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 1b8f1ea215e8c2f78186b80dafed1b28 2024-11-15T03:13:44,153 INFO [StoreOpener-1b8f1ea215e8c2f78186b80dafed1b28-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1b8f1ea215e8c2f78186b80dafed1b28 columnFamilyName family 2024-11-15T03:13:44,153 DEBUG [StoreOpener-1b8f1ea215e8c2f78186b80dafed1b28-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T03:13:44,154 INFO [StoreOpener-1b8f1ea215e8c2f78186b80dafed1b28-1 {}] regionserver.HStore(327): Store=1b8f1ea215e8c2f78186b80dafed1b28/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T03:13:44,154 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for 1b8f1ea215e8c2f78186b80dafed1b28 2024-11-15T03:13:44,155 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/1b8f1ea215e8c2f78186b80dafed1b28 2024-11-15T03:13:44,156 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/1b8f1ea215e8c2f78186b80dafed1b28 2024-11-15T03:13:44,156 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for 1b8f1ea215e8c2f78186b80dafed1b28 2024-11-15T03:13:44,156 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for 1b8f1ea215e8c2f78186b80dafed1b28 2024-11-15T03:13:44,158 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for 1b8f1ea215e8c2f78186b80dafed1b28 2024-11-15T03:13:44,161 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/1b8f1ea215e8c2f78186b80dafed1b28/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T03:13:44,162 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened 1b8f1ea215e8c2f78186b80dafed1b28; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60134711, jitterRate=-0.10392297804355621}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T03:13:44,162 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1b8f1ea215e8c2f78186b80dafed1b28 2024-11-15T03:13:44,163 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for 1b8f1ea215e8c2f78186b80dafed1b28: Running coprocessor pre-open hook at 1731640424149Writing region info on filesystem at 1731640424149Initializing all the Stores at 1731640424151 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731640424151Cleaning up temporary data from old regions at 1731640424156 (+5 ms)Running coprocessor post-open hooks at 1731640424162 (+6 ms)Region opened successfully at 1731640424163 (+1 ms) 2024-11-15T03:13:44,165 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for test,uuu,1731640423069.1b8f1ea215e8c2f78186b80dafed1b28., pid=35, masterSystemTime=1731640424140 2024-11-15T03:13:44,169 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for test,uuu,1731640423069.1b8f1ea215e8c2f78186b80dafed1b28. 2024-11-15T03:13:44,169 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened test,uuu,1731640423069.1b8f1ea215e8c2f78186b80dafed1b28. 2024-11-15T03:13:44,169 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open test,qqq,1731640423069.23f804d67a47ba185ffd6f6bc7b75464. 2024-11-15T03:13:44,169 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 23f804d67a47ba185ffd6f6bc7b75464, NAME => 'test,qqq,1731640423069.23f804d67a47ba185ffd6f6bc7b75464.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-11-15T03:13:44,170 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 23f804d67a47ba185ffd6f6bc7b75464 2024-11-15T03:13:44,170 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated test,qqq,1731640423069.23f804d67a47ba185ffd6f6bc7b75464.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T03:13:44,170 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 23f804d67a47ba185ffd6f6bc7b75464 2024-11-15T03:13:44,170 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=1b8f1ea215e8c2f78186b80dafed1b28, regionState=OPEN, openSeqNum=2, regionLocation=91f82191efb8,42027,1731640420387 2024-11-15T03:13:44,170 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 23f804d67a47ba185ffd6f6bc7b75464 2024-11-15T03:13:44,173 INFO [StoreOpener-23f804d67a47ba185ffd6f6bc7b75464-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 23f804d67a47ba185ffd6f6bc7b75464 2024-11-15T03:13:44,174 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1b8f1ea215e8c2f78186b80dafed1b28, server=91f82191efb8,42027,1731640420387 because future has completed 2024-11-15T03:13:44,176 INFO [StoreOpener-23f804d67a47ba185ffd6f6bc7b75464-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 23f804d67a47ba185ffd6f6bc7b75464 columnFamilyName family 2024-11-15T03:13:44,176 DEBUG [StoreOpener-23f804d67a47ba185ffd6f6bc7b75464-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T03:13:44,177 INFO [StoreOpener-23f804d67a47ba185ffd6f6bc7b75464-1 {}] regionserver.HStore(327): Store=23f804d67a47ba185ffd6f6bc7b75464/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T03:13:44,177 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 23f804d67a47ba185ffd6f6bc7b75464 2024-11-15T03:13:44,178 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/23f804d67a47ba185ffd6f6bc7b75464 2024-11-15T03:13:44,179 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=25 2024-11-15T03:13:44,179 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure 1b8f1ea215e8c2f78186b80dafed1b28, server=91f82191efb8,42027,1731640420387 in 158 msec 2024-11-15T03:13:44,179 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/23f804d67a47ba185ffd6f6bc7b75464 2024-11-15T03:13:44,180 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 23f804d67a47ba185ffd6f6bc7b75464 2024-11-15T03:13:44,180 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 23f804d67a47ba185ffd6f6bc7b75464 2024-11-15T03:13:44,182 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 23f804d67a47ba185ffd6f6bc7b75464 2024-11-15T03:13:44,182 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=1b8f1ea215e8c2f78186b80dafed1b28, ASSIGN in 368 msec 2024-11-15T03:13:44,185 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/23f804d67a47ba185ffd6f6bc7b75464/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T03:13:44,186 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 23f804d67a47ba185ffd6f6bc7b75464; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67657959, jitterRate=0.008182153105735779}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T03:13:44,186 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 23f804d67a47ba185ffd6f6bc7b75464 2024-11-15T03:13:44,186 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 23f804d67a47ba185ffd6f6bc7b75464: Running coprocessor pre-open hook at 1731640424170Writing region info on filesystem at 1731640424170Initializing all the Stores at 1731640424172 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731640424172Cleaning up temporary data from old regions at 1731640424180 (+8 ms)Running coprocessor post-open hooks at 1731640424186 (+6 ms)Region opened successfully at 1731640424186 2024-11-15T03:13:44,187 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for test,qqq,1731640423069.23f804d67a47ba185ffd6f6bc7b75464., pid=32, masterSystemTime=1731640424140 2024-11-15T03:13:44,190 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for test,qqq,1731640423069.23f804d67a47ba185ffd6f6bc7b75464. 2024-11-15T03:13:44,190 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened test,qqq,1731640423069.23f804d67a47ba185ffd6f6bc7b75464. 2024-11-15T03:13:44,190 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open test,ddd,1731640423069.1beb579fcf8e161a0b9c2051c671a8f2. 2024-11-15T03:13:44,191 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => 1beb579fcf8e161a0b9c2051c671a8f2, NAME => 'test,ddd,1731640423069.1beb579fcf8e161a0b9c2051c671a8f2.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-11-15T03:13:44,191 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 1beb579fcf8e161a0b9c2051c671a8f2 2024-11-15T03:13:44,191 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=23f804d67a47ba185ffd6f6bc7b75464, regionState=OPEN, openSeqNum=2, regionLocation=91f82191efb8,42027,1731640420387 2024-11-15T03:13:44,191 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated test,ddd,1731640423069.1beb579fcf8e161a0b9c2051c671a8f2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T03:13:44,191 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for 1beb579fcf8e161a0b9c2051c671a8f2 2024-11-15T03:13:44,191 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for 1beb579fcf8e161a0b9c2051c671a8f2 2024-11-15T03:13:44,194 INFO [StoreOpener-1beb579fcf8e161a0b9c2051c671a8f2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 1beb579fcf8e161a0b9c2051c671a8f2 2024-11-15T03:13:44,194 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 23f804d67a47ba185ffd6f6bc7b75464, server=91f82191efb8,42027,1731640420387 because future has completed 2024-11-15T03:13:44,196 INFO [StoreOpener-1beb579fcf8e161a0b9c2051c671a8f2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1beb579fcf8e161a0b9c2051c671a8f2 columnFamilyName family 2024-11-15T03:13:44,196 DEBUG [StoreOpener-1beb579fcf8e161a0b9c2051c671a8f2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T03:13:44,197 INFO [StoreOpener-1beb579fcf8e161a0b9c2051c671a8f2-1 {}] regionserver.HStore(327): Store=1beb579fcf8e161a0b9c2051c671a8f2/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T03:13:44,197 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for 1beb579fcf8e161a0b9c2051c671a8f2 2024-11-15T03:13:44,198 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=21 2024-11-15T03:13:44,198 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=21, state=SUCCESS, hasLock=false; OpenRegionProcedure 23f804d67a47ba185ffd6f6bc7b75464, server=91f82191efb8,42027,1731640420387 in 200 msec 2024-11-15T03:13:44,198 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/1beb579fcf8e161a0b9c2051c671a8f2 2024-11-15T03:13:44,199 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/1beb579fcf8e161a0b9c2051c671a8f2 2024-11-15T03:13:44,199 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-15T03:13:44,200 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for 1beb579fcf8e161a0b9c2051c671a8f2 2024-11-15T03:13:44,200 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for 1beb579fcf8e161a0b9c2051c671a8f2 2024-11-15T03:13:44,201 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43381, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-15T03:13:44,201 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=23f804d67a47ba185ffd6f6bc7b75464, ASSIGN in 387 msec 2024-11-15T03:13:44,203 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for 1beb579fcf8e161a0b9c2051c671a8f2 2024-11-15T03:13:44,206 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/1beb579fcf8e161a0b9c2051c671a8f2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T03:13:44,206 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open test,ooo,1731640423069.b96380a8d7110dffa8cd7844252025dd. 2024-11-15T03:13:44,207 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => b96380a8d7110dffa8cd7844252025dd, NAME => 'test,ooo,1731640423069.b96380a8d7110dffa8cd7844252025dd.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-11-15T03:13:44,207 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened 1beb579fcf8e161a0b9c2051c671a8f2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59170768, jitterRate=-0.1182868480682373}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T03:13:44,207 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1beb579fcf8e161a0b9c2051c671a8f2 2024-11-15T03:13:44,207 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for 1beb579fcf8e161a0b9c2051c671a8f2: Running coprocessor pre-open hook at 1731640424191Writing region info on filesystem at 1731640424191Initializing all the Stores at 1731640424194 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731640424194Cleaning up temporary data from old regions at 1731640424200 (+6 ms)Running coprocessor post-open hooks at 1731640424207 (+7 ms)Region opened successfully at 1731640424207 2024-11-15T03:13:44,207 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test b96380a8d7110dffa8cd7844252025dd 2024-11-15T03:13:44,207 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated test,ooo,1731640423069.b96380a8d7110dffa8cd7844252025dd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T03:13:44,208 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for b96380a8d7110dffa8cd7844252025dd 2024-11-15T03:13:44,208 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for b96380a8d7110dffa8cd7844252025dd 2024-11-15T03:13:44,208 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ddd,1731640423069.1beb579fcf8e161a0b9c2051c671a8f2., pid=43, masterSystemTime=1731640424140 2024-11-15T03:13:44,210 INFO [StoreOpener-b96380a8d7110dffa8cd7844252025dd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region b96380a8d7110dffa8cd7844252025dd 2024-11-15T03:13:44,211 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ddd,1731640423069.1beb579fcf8e161a0b9c2051c671a8f2. 2024-11-15T03:13:44,211 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened test,ddd,1731640423069.1beb579fcf8e161a0b9c2051c671a8f2. 2024-11-15T03:13:44,212 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open test,kkk,1731640423069.bb2ead3b463edc0f2b603da7d19f1201. 2024-11-15T03:13:44,212 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => bb2ead3b463edc0f2b603da7d19f1201, NAME => 'test,kkk,1731640423069.bb2ead3b463edc0f2b603da7d19f1201.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-11-15T03:13:44,213 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test bb2ead3b463edc0f2b603da7d19f1201 2024-11-15T03:13:44,213 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated test,kkk,1731640423069.bb2ead3b463edc0f2b603da7d19f1201.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T03:13:44,213 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for bb2ead3b463edc0f2b603da7d19f1201 2024-11-15T03:13:44,213 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for bb2ead3b463edc0f2b603da7d19f1201 2024-11-15T03:13:44,214 INFO [StoreOpener-b96380a8d7110dffa8cd7844252025dd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b96380a8d7110dffa8cd7844252025dd columnFamilyName family 2024-11-15T03:13:44,214 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1beb579fcf8e161a0b9c2051c671a8f2, regionState=OPEN, openSeqNum=2, regionLocation=91f82191efb8,42027,1731640420387 2024-11-15T03:13:44,214 DEBUG [StoreOpener-b96380a8d7110dffa8cd7844252025dd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T03:13:44,215 INFO [StoreOpener-b96380a8d7110dffa8cd7844252025dd-1 {}] regionserver.HStore(327): Store=b96380a8d7110dffa8cd7844252025dd/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T03:13:44,215 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for b96380a8d7110dffa8cd7844252025dd 2024-11-15T03:13:44,216 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/b96380a8d7110dffa8cd7844252025dd 2024-11-15T03:13:44,218 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1beb579fcf8e161a0b9c2051c671a8f2, server=91f82191efb8,42027,1731640420387 because future has completed 2024-11-15T03:13:44,218 INFO [StoreOpener-bb2ead3b463edc0f2b603da7d19f1201-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region bb2ead3b463edc0f2b603da7d19f1201 2024-11-15T03:13:44,218 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/b96380a8d7110dffa8cd7844252025dd 2024-11-15T03:13:44,219 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for b96380a8d7110dffa8cd7844252025dd 2024-11-15T03:13:44,219 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42051 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=91f82191efb8,42027,1731640420387, table=test, region=1beb579fcf8e161a0b9c2051c671a8f2. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T03:13:44,219 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for b96380a8d7110dffa8cd7844252025dd 2024-11-15T03:13:44,221 INFO [StoreOpener-bb2ead3b463edc0f2b603da7d19f1201-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bb2ead3b463edc0f2b603da7d19f1201 columnFamilyName family 2024-11-15T03:13:44,222 DEBUG [StoreOpener-bb2ead3b463edc0f2b603da7d19f1201-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T03:13:44,223 INFO [StoreOpener-bb2ead3b463edc0f2b603da7d19f1201-1 {}] regionserver.HStore(327): Store=bb2ead3b463edc0f2b603da7d19f1201/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T03:13:44,223 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for bb2ead3b463edc0f2b603da7d19f1201 2024-11-15T03:13:44,224 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/bb2ead3b463edc0f2b603da7d19f1201 2024-11-15T03:13:44,224 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=8 2024-11-15T03:13:44,225 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 1beb579fcf8e161a0b9c2051c671a8f2, server=91f82191efb8,42027,1731640420387 in 154 msec 2024-11-15T03:13:44,225 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/bb2ead3b463edc0f2b603da7d19f1201 2024-11-15T03:13:44,226 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for b96380a8d7110dffa8cd7844252025dd 2024-11-15T03:13:44,226 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for bb2ead3b463edc0f2b603da7d19f1201 2024-11-15T03:13:44,226 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for bb2ead3b463edc0f2b603da7d19f1201 2024-11-15T03:13:44,228 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=1beb579fcf8e161a0b9c2051c671a8f2, ASSIGN in 414 msec 2024-11-15T03:13:44,229 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for bb2ead3b463edc0f2b603da7d19f1201 2024-11-15T03:13:44,229 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/b96380a8d7110dffa8cd7844252025dd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T03:13:44,230 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened b96380a8d7110dffa8cd7844252025dd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63358111, jitterRate=-0.05589057505130768}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T03:13:44,230 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b96380a8d7110dffa8cd7844252025dd 2024-11-15T03:13:44,230 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for b96380a8d7110dffa8cd7844252025dd: Running coprocessor pre-open hook at 1731640424208Writing region info on filesystem at 1731640424208Initializing all the Stores at 1731640424209 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731640424209Cleaning up temporary data from old regions at 1731640424219 (+10 ms)Running coprocessor post-open hooks at 1731640424230 (+11 ms)Region opened successfully at 1731640424230 2024-11-15T03:13:44,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T03:13:44,232 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ooo,1731640423069.b96380a8d7110dffa8cd7844252025dd., pid=40, masterSystemTime=1731640424199 2024-11-15T03:13:44,233 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/bb2ead3b463edc0f2b603da7d19f1201/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T03:13:44,234 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-15T03:13:44,234 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened bb2ead3b463edc0f2b603da7d19f1201; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73290503, jitterRate=0.09211359918117523}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T03:13:44,234 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bb2ead3b463edc0f2b603da7d19f1201 2024-11-15T03:13:44,235 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for bb2ead3b463edc0f2b603da7d19f1201: Running coprocessor pre-open hook at 1731640424213Writing region info on filesystem at 1731640424213Initializing all the Stores at 1731640424215 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731640424215Cleaning up temporary data from old regions at 1731640424226 (+11 ms)Running coprocessor post-open hooks at 1731640424234 (+8 ms)Region opened successfully at 1731640424234 2024-11-15T03:13:44,236 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for test,kkk,1731640423069.bb2ead3b463edc0f2b603da7d19f1201., pid=33, masterSystemTime=1731640424140 2024-11-15T03:13:44,237 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ooo,1731640423069.b96380a8d7110dffa8cd7844252025dd. 2024-11-15T03:13:44,238 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened test,ooo,1731640423069.b96380a8d7110dffa8cd7844252025dd. 2024-11-15T03:13:44,238 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43249, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-15T03:13:44,238 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(132): Open test,xxx,1731640423069.6cca2d3f6a001d96f34b7a93afec8fe7. 2024-11-15T03:13:44,238 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7752): Opening region: {ENCODED => 6cca2d3f6a001d96f34b7a93afec8fe7, NAME => 'test,xxx,1731640423069.6cca2d3f6a001d96f34b7a93afec8fe7.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-11-15T03:13:44,238 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 6cca2d3f6a001d96f34b7a93afec8fe7 2024-11-15T03:13:44,238 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(898): Instantiated test,xxx,1731640423069.6cca2d3f6a001d96f34b7a93afec8fe7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T03:13:44,238 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7794): checking encryption for 6cca2d3f6a001d96f34b7a93afec8fe7 2024-11-15T03:13:44,239 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=b96380a8d7110dffa8cd7844252025dd, regionState=OPEN, openSeqNum=2, regionLocation=91f82191efb8,41451,1731640420638 2024-11-15T03:13:44,240 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for test,kkk,1731640423069.bb2ead3b463edc0f2b603da7d19f1201. 2024-11-15T03:13:44,240 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened test,kkk,1731640423069.bb2ead3b463edc0f2b603da7d19f1201. 2024-11-15T03:13:44,240 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7797): checking classloading for 6cca2d3f6a001d96f34b7a93afec8fe7 2024-11-15T03:13:44,240 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open test,yyy,1731640423069.e0bce1a97a9c7f320b315db443b47c23. 2024-11-15T03:13:44,240 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => e0bce1a97a9c7f320b315db443b47c23, NAME => 'test,yyy,1731640423069.e0bce1a97a9c7f320b315db443b47c23.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-15T03:13:44,241 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test e0bce1a97a9c7f320b315db443b47c23 2024-11-15T03:13:44,241 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated test,yyy,1731640423069.e0bce1a97a9c7f320b315db443b47c23.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T03:13:44,241 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for e0bce1a97a9c7f320b315db443b47c23 2024-11-15T03:13:44,241 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for e0bce1a97a9c7f320b315db443b47c23 2024-11-15T03:13:44,244 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure b96380a8d7110dffa8cd7844252025dd, server=91f82191efb8,41451,1731640420638 because future has completed 2024-11-15T03:13:44,245 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=bb2ead3b463edc0f2b603da7d19f1201, regionState=OPEN, openSeqNum=2, regionLocation=91f82191efb8,42027,1731640420387 2024-11-15T03:13:44,248 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,42027,1731640420387 because future has completed 2024-11-15T03:13:44,249 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(132): Open test,ttt,1731640423069.21df709a703798d68290a0a63322957f. 2024-11-15T03:13:44,249 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7752): Opening region: {ENCODED => 21df709a703798d68290a0a63322957f, NAME => 'test,ttt,1731640423069.21df709a703798d68290a0a63322957f.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-15T03:13:44,249 INFO [StoreOpener-e0bce1a97a9c7f320b315db443b47c23-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region e0bce1a97a9c7f320b315db443b47c23 2024-11-15T03:13:44,250 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 21df709a703798d68290a0a63322957f 2024-11-15T03:13:44,250 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(898): Instantiated test,ttt,1731640423069.21df709a703798d68290a0a63322957f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T03:13:44,250 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7794): checking encryption for 21df709a703798d68290a0a63322957f 2024-11-15T03:13:44,250 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7797): checking classloading for 21df709a703798d68290a0a63322957f 2024-11-15T03:13:44,252 INFO [StoreOpener-e0bce1a97a9c7f320b315db443b47c23-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e0bce1a97a9c7f320b315db443b47c23 columnFamilyName family 2024-11-15T03:13:44,252 DEBUG [StoreOpener-e0bce1a97a9c7f320b315db443b47c23-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T03:13:44,253 INFO [StoreOpener-21df709a703798d68290a0a63322957f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 21df709a703798d68290a0a63322957f 2024-11-15T03:13:44,253 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=19 2024-11-15T03:13:44,253 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=19, state=SUCCESS, hasLock=false; OpenRegionProcedure b96380a8d7110dffa8cd7844252025dd, server=91f82191efb8,41451,1731640420638 in 196 msec 2024-11-15T03:13:44,253 INFO [StoreOpener-e0bce1a97a9c7f320b315db443b47c23-1 {}] regionserver.HStore(327): Store=e0bce1a97a9c7f320b315db443b47c23/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T03:13:44,253 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for e0bce1a97a9c7f320b315db443b47c23 2024-11-15T03:13:44,254 INFO [StoreOpener-6cca2d3f6a001d96f34b7a93afec8fe7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 6cca2d3f6a001d96f34b7a93afec8fe7 2024-11-15T03:13:44,255 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/e0bce1a97a9c7f320b315db443b47c23 2024-11-15T03:13:44,255 INFO [StoreOpener-21df709a703798d68290a0a63322957f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 21df709a703798d68290a0a63322957f columnFamilyName family 2024-11-15T03:13:44,255 DEBUG [StoreOpener-21df709a703798d68290a0a63322957f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T03:13:44,255 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/e0bce1a97a9c7f320b315db443b47c23 2024-11-15T03:13:44,256 INFO [StoreOpener-21df709a703798d68290a0a63322957f-1 {}] regionserver.HStore(327): Store=21df709a703798d68290a0a63322957f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T03:13:44,256 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for e0bce1a97a9c7f320b315db443b47c23 2024-11-15T03:13:44,256 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for e0bce1a97a9c7f320b315db443b47c23 2024-11-15T03:13:44,256 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1038): replaying wal for 21df709a703798d68290a0a63322957f 2024-11-15T03:13:44,256 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=b96380a8d7110dffa8cd7844252025dd, ASSIGN in 442 msec 2024-11-15T03:13:44,257 INFO [StoreOpener-6cca2d3f6a001d96f34b7a93afec8fe7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6cca2d3f6a001d96f34b7a93afec8fe7 columnFamilyName family 2024-11-15T03:13:44,257 DEBUG [StoreOpener-6cca2d3f6a001d96f34b7a93afec8fe7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T03:13:44,258 INFO [StoreOpener-6cca2d3f6a001d96f34b7a93afec8fe7-1 {}] regionserver.HStore(327): Store=6cca2d3f6a001d96f34b7a93afec8fe7/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T03:13:44,258 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/21df709a703798d68290a0a63322957f 2024-11-15T03:13:44,258 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1038): replaying wal for 6cca2d3f6a001d96f34b7a93afec8fe7 2024-11-15T03:13:44,259 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/21df709a703798d68290a0a63322957f 2024-11-15T03:13:44,260 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1048): stopping wal replay for 21df709a703798d68290a0a63322957f 2024-11-15T03:13:44,260 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1060): Cleaning up temporary data for 21df709a703798d68290a0a63322957f 2024-11-15T03:13:44,260 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/6cca2d3f6a001d96f34b7a93afec8fe7 2024-11-15T03:13:44,260 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for e0bce1a97a9c7f320b315db443b47c23 2024-11-15T03:13:44,261 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/6cca2d3f6a001d96f34b7a93afec8fe7 2024-11-15T03:13:44,261 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1048): stopping wal replay for 6cca2d3f6a001d96f34b7a93afec8fe7 2024-11-15T03:13:44,262 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1060): Cleaning up temporary data for 6cca2d3f6a001d96f34b7a93afec8fe7 2024-11-15T03:13:44,265 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=15 2024-11-15T03:13:44,265 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,42027,1731640420387 in 254 msec 2024-11-15T03:13:44,265 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1093): writing seq id for 21df709a703798d68290a0a63322957f 2024-11-15T03:13:44,265 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/e0bce1a97a9c7f320b315db443b47c23/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T03:13:44,265 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1093): writing seq id for 6cca2d3f6a001d96f34b7a93afec8fe7 2024-11-15T03:13:44,266 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened e0bce1a97a9c7f320b315db443b47c23; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74519135, jitterRate=0.11042164266109467}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T03:13:44,266 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e0bce1a97a9c7f320b315db443b47c23 2024-11-15T03:13:44,267 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for e0bce1a97a9c7f320b315db443b47c23: Running coprocessor pre-open hook at 1731640424242Writing region info on filesystem at 1731640424242Initializing all the Stores at 1731640424243 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731640424243Cleaning up temporary data from old regions at 1731640424256 (+13 ms)Running coprocessor post-open hooks at 1731640424266 (+10 ms)Region opened successfully at 1731640424266 2024-11-15T03:13:44,268 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1731640423069.e0bce1a97a9c7f320b315db443b47c23., pid=31, masterSystemTime=1731640424140 2024-11-15T03:13:44,270 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=bb2ead3b463edc0f2b603da7d19f1201, ASSIGN in 454 msec 2024-11-15T03:13:44,271 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1731640423069.e0bce1a97a9c7f320b315db443b47c23. 2024-11-15T03:13:44,271 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened test,yyy,1731640423069.e0bce1a97a9c7f320b315db443b47c23. 2024-11-15T03:13:44,271 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open test,nnn,1731640423069.a73d2d096e2f14a7e9365ac762927d55. 2024-11-15T03:13:44,272 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => a73d2d096e2f14a7e9365ac762927d55, NAME => 'test,nnn,1731640423069.a73d2d096e2f14a7e9365ac762927d55.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-11-15T03:13:44,272 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test a73d2d096e2f14a7e9365ac762927d55 2024-11-15T03:13:44,272 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated test,nnn,1731640423069.a73d2d096e2f14a7e9365ac762927d55.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T03:13:44,272 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for a73d2d096e2f14a7e9365ac762927d55 2024-11-15T03:13:44,272 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for a73d2d096e2f14a7e9365ac762927d55 2024-11-15T03:13:44,272 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=e0bce1a97a9c7f320b315db443b47c23, regionState=OPEN, openSeqNum=2, regionLocation=91f82191efb8,42027,1731640420387 2024-11-15T03:13:44,275 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure e0bce1a97a9c7f320b315db443b47c23, server=91f82191efb8,42027,1731640420387 because future has completed 2024-11-15T03:13:44,277 INFO [StoreOpener-a73d2d096e2f14a7e9365ac762927d55-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region a73d2d096e2f14a7e9365ac762927d55 2024-11-15T03:13:44,277 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/6cca2d3f6a001d96f34b7a93afec8fe7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T03:13:44,277 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/21df709a703798d68290a0a63322957f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T03:13:44,278 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1114): Opened 6cca2d3f6a001d96f34b7a93afec8fe7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59689139, jitterRate=-0.1105625182390213}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T03:13:44,278 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6cca2d3f6a001d96f34b7a93afec8fe7 2024-11-15T03:13:44,278 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1006): Region open journal for 6cca2d3f6a001d96f34b7a93afec8fe7: Running coprocessor pre-open hook at 1731640424240Writing region info on filesystem at 1731640424240Initializing all the Stores at 1731640424243 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731640424243Cleaning up temporary data from old regions at 1731640424262 (+19 ms)Running coprocessor post-open hooks at 1731640424278 (+16 ms)Region opened successfully at 1731640424278 2024-11-15T03:13:44,278 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1114): Opened 21df709a703798d68290a0a63322957f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61527944, jitterRate=-0.08316218852996826}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T03:13:44,279 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 21df709a703798d68290a0a63322957f 2024-11-15T03:13:44,279 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1006): Region open journal for 21df709a703798d68290a0a63322957f: Running coprocessor pre-open hook at 1731640424250Writing region info on filesystem at 1731640424250Initializing all the Stores at 1731640424252 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731640424252Cleaning up temporary data from old regions at 1731640424260 (+8 ms)Running coprocessor post-open hooks at 1731640424279 (+19 ms)Region opened successfully at 1731640424279 2024-11-15T03:13:44,279 INFO [StoreOpener-a73d2d096e2f14a7e9365ac762927d55-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a73d2d096e2f14a7e9365ac762927d55 columnFamilyName family 2024-11-15T03:13:44,280 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1731640423069.6cca2d3f6a001d96f34b7a93afec8fe7., pid=42, masterSystemTime=1731640424199 2024-11-15T03:13:44,280 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1731640423069.21df709a703798d68290a0a63322957f., pid=54, masterSystemTime=1731640424234 2024-11-15T03:13:44,280 DEBUG [StoreOpener-a73d2d096e2f14a7e9365ac762927d55-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T03:13:44,281 INFO [StoreOpener-a73d2d096e2f14a7e9365ac762927d55-1 {}] regionserver.HStore(327): Store=a73d2d096e2f14a7e9365ac762927d55/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T03:13:44,281 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for a73d2d096e2f14a7e9365ac762927d55 2024-11-15T03:13:44,282 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=29 2024-11-15T03:13:44,282 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure e0bce1a97a9c7f320b315db443b47c23, server=91f82191efb8,42027,1731640420387 in 293 msec 2024-11-15T03:13:44,283 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/a73d2d096e2f14a7e9365ac762927d55 2024-11-15T03:13:44,283 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1731640423069.21df709a703798d68290a0a63322957f. 2024-11-15T03:13:44,283 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/a73d2d096e2f14a7e9365ac762927d55 2024-11-15T03:13:44,283 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(153): Opened test,ttt,1731640423069.21df709a703798d68290a0a63322957f. 2024-11-15T03:13:44,284 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open test,fff,1731640423069.c4b611da4a73d4cfc99379713486e21c. 2024-11-15T03:13:44,284 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => c4b611da4a73d4cfc99379713486e21c, NAME => 'test,fff,1731640423069.c4b611da4a73d4cfc99379713486e21c.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-15T03:13:44,284 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=21df709a703798d68290a0a63322957f, regionState=OPEN, openSeqNum=2, regionLocation=91f82191efb8,37523,1731640420593 2024-11-15T03:13:44,284 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for a73d2d096e2f14a7e9365ac762927d55 2024-11-15T03:13:44,284 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for a73d2d096e2f14a7e9365ac762927d55 2024-11-15T03:13:44,284 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c4b611da4a73d4cfc99379713486e21c 2024-11-15T03:13:44,285 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated test,fff,1731640423069.c4b611da4a73d4cfc99379713486e21c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T03:13:44,285 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for c4b611da4a73d4cfc99379713486e21c 2024-11-15T03:13:44,285 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1731640423069.6cca2d3f6a001d96f34b7a93afec8fe7. 2024-11-15T03:13:44,285 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(153): Opened test,xxx,1731640423069.6cca2d3f6a001d96f34b7a93afec8fe7. 2024-11-15T03:13:44,285 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(132): Open test,iii,1731640423069.f5219cfe492c74eb8312930d328964b6. 2024-11-15T03:13:44,285 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7752): Opening region: {ENCODED => f5219cfe492c74eb8312930d328964b6, NAME => 'test,iii,1731640423069.f5219cfe492c74eb8312930d328964b6.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-15T03:13:44,286 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f5219cfe492c74eb8312930d328964b6 2024-11-15T03:13:44,286 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for c4b611da4a73d4cfc99379713486e21c 2024-11-15T03:13:44,286 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(898): Instantiated test,iii,1731640423069.f5219cfe492c74eb8312930d328964b6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T03:13:44,287 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7794): checking encryption for f5219cfe492c74eb8312930d328964b6 2024-11-15T03:13:44,287 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7797): checking classloading for f5219cfe492c74eb8312930d328964b6 2024-11-15T03:13:44,288 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for a73d2d096e2f14a7e9365ac762927d55 2024-11-15T03:13:44,288 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 21df709a703798d68290a0a63322957f, server=91f82191efb8,37523,1731640420593 because future has completed 2024-11-15T03:13:44,289 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=6cca2d3f6a001d96f34b7a93afec8fe7, regionState=OPEN, openSeqNum=2, regionLocation=91f82191efb8,41451,1731640420638 2024-11-15T03:13:44,289 INFO [StoreOpener-c4b611da4a73d4cfc99379713486e21c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c4b611da4a73d4cfc99379713486e21c 2024-11-15T03:13:44,292 INFO [StoreOpener-c4b611da4a73d4cfc99379713486e21c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c4b611da4a73d4cfc99379713486e21c columnFamilyName family 2024-11-15T03:13:44,292 DEBUG [StoreOpener-c4b611da4a73d4cfc99379713486e21c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T03:13:44,293 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/a73d2d096e2f14a7e9365ac762927d55/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T03:13:44,293 INFO [StoreOpener-c4b611da4a73d4cfc99379713486e21c-1 {}] regionserver.HStore(327): Store=c4b611da4a73d4cfc99379713486e21c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T03:13:44,293 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for c4b611da4a73d4cfc99379713486e21c 2024-11-15T03:13:44,294 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6cca2d3f6a001d96f34b7a93afec8fe7, server=91f82191efb8,41451,1731640420638 because future has completed 2024-11-15T03:13:44,296 INFO [StoreOpener-f5219cfe492c74eb8312930d328964b6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f5219cfe492c74eb8312930d328964b6 2024-11-15T03:13:44,296 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/c4b611da4a73d4cfc99379713486e21c 2024-11-15T03:13:44,296 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened a73d2d096e2f14a7e9365ac762927d55; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64858928, jitterRate=-0.03352665901184082}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T03:13:44,297 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a73d2d096e2f14a7e9365ac762927d55 2024-11-15T03:13:44,297 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for a73d2d096e2f14a7e9365ac762927d55: Running coprocessor pre-open hook at 1731640424272Writing region info on filesystem at 1731640424272Initializing all the Stores at 1731640424274 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731640424274Cleaning up temporary data from old regions at 1731640424284 (+10 ms)Running coprocessor post-open hooks at 1731640424297 (+13 ms)Region opened successfully at 1731640424297 2024-11-15T03:13:44,298 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for test,nnn,1731640423069.a73d2d096e2f14a7e9365ac762927d55., pid=34, masterSystemTime=1731640424140 2024-11-15T03:13:44,299 INFO [StoreOpener-f5219cfe492c74eb8312930d328964b6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f5219cfe492c74eb8312930d328964b6 columnFamilyName family 2024-11-15T03:13:44,299 DEBUG [StoreOpener-f5219cfe492c74eb8312930d328964b6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T03:13:44,300 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/c4b611da4a73d4cfc99379713486e21c 2024-11-15T03:13:44,301 INFO [StoreOpener-f5219cfe492c74eb8312930d328964b6-1 {}] regionserver.HStore(327): Store=f5219cfe492c74eb8312930d328964b6/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T03:13:44,301 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1038): replaying wal for f5219cfe492c74eb8312930d328964b6 2024-11-15T03:13:44,301 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for test,nnn,1731640423069.a73d2d096e2f14a7e9365ac762927d55. 2024-11-15T03:13:44,301 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened test,nnn,1731640423069.a73d2d096e2f14a7e9365ac762927d55. 2024-11-15T03:13:44,301 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for c4b611da4a73d4cfc99379713486e21c 2024-11-15T03:13:44,301 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for c4b611da4a73d4cfc99379713486e21c 2024-11-15T03:13:44,302 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open test,ggg,1731640423069.d0eff1b91bc9b77579a5c44f1c985985. 2024-11-15T03:13:44,302 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => d0eff1b91bc9b77579a5c44f1c985985, NAME => 'test,ggg,1731640423069.d0eff1b91bc9b77579a5c44f1c985985.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-11-15T03:13:44,302 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d0eff1b91bc9b77579a5c44f1c985985 2024-11-15T03:13:44,302 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated test,ggg,1731640423069.d0eff1b91bc9b77579a5c44f1c985985.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T03:13:44,302 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for d0eff1b91bc9b77579a5c44f1c985985 2024-11-15T03:13:44,302 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for d0eff1b91bc9b77579a5c44f1c985985 2024-11-15T03:13:44,304 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=28 2024-11-15T03:13:44,304 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/f5219cfe492c74eb8312930d328964b6 2024-11-15T03:13:44,304 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=28, state=SUCCESS, hasLock=false; OpenRegionProcedure 6cca2d3f6a001d96f34b7a93afec8fe7, server=91f82191efb8,41451,1731640420638 in 238 msec 2024-11-15T03:13:44,304 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/f5219cfe492c74eb8312930d328964b6 2024-11-15T03:13:44,305 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1048): stopping wal replay for f5219cfe492c74eb8312930d328964b6 2024-11-15T03:13:44,305 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1060): Cleaning up temporary data for f5219cfe492c74eb8312930d328964b6 2024-11-15T03:13:44,306 INFO [StoreOpener-d0eff1b91bc9b77579a5c44f1c985985-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d0eff1b91bc9b77579a5c44f1c985985 2024-11-15T03:13:44,306 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for c4b611da4a73d4cfc99379713486e21c 2024-11-15T03:13:44,308 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6cca2d3f6a001d96f34b7a93afec8fe7, ASSIGN in 493 msec 2024-11-15T03:13:44,308 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1093): writing seq id for f5219cfe492c74eb8312930d328964b6 2024-11-15T03:13:44,309 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/c4b611da4a73d4cfc99379713486e21c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T03:13:44,309 INFO [StoreOpener-d0eff1b91bc9b77579a5c44f1c985985-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d0eff1b91bc9b77579a5c44f1c985985 columnFamilyName family 2024-11-15T03:13:44,310 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=a73d2d096e2f14a7e9365ac762927d55, regionState=OPEN, openSeqNum=2, regionLocation=91f82191efb8,42027,1731640420387 2024-11-15T03:13:44,310 DEBUG [StoreOpener-d0eff1b91bc9b77579a5c44f1c985985-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T03:13:44,310 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened c4b611da4a73d4cfc99379713486e21c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59945231, jitterRate=-0.10674645006656647}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T03:13:44,310 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c4b611da4a73d4cfc99379713486e21c 2024-11-15T03:13:44,310 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for c4b611da4a73d4cfc99379713486e21c: Running coprocessor pre-open hook at 1731640424286Writing region info on filesystem at 1731640424286Initializing all the Stores at 1731640424288 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731640424288Cleaning up temporary data from old regions at 1731640424302 (+14 ms)Running coprocessor post-open hooks at 1731640424310 (+8 ms)Region opened successfully at 1731640424310 2024-11-15T03:13:44,312 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1731640423069.c4b611da4a73d4cfc99379713486e21c., pid=55, masterSystemTime=1731640424234 2024-11-15T03:13:44,312 INFO [StoreOpener-d0eff1b91bc9b77579a5c44f1c985985-1 {}] regionserver.HStore(327): Store=d0eff1b91bc9b77579a5c44f1c985985/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T03:13:44,313 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for d0eff1b91bc9b77579a5c44f1c985985 2024-11-15T03:13:44,313 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/f5219cfe492c74eb8312930d328964b6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T03:13:44,313 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure a73d2d096e2f14a7e9365ac762927d55, server=91f82191efb8,42027,1731640420387 because future has completed 2024-11-15T03:13:44,314 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/d0eff1b91bc9b77579a5c44f1c985985 2024-11-15T03:13:44,314 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1114): Opened f5219cfe492c74eb8312930d328964b6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59809282, jitterRate=-0.10877224802970886}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T03:13:44,314 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f5219cfe492c74eb8312930d328964b6 2024-11-15T03:13:44,315 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1006): Region open journal for f5219cfe492c74eb8312930d328964b6: Running coprocessor pre-open hook at 1731640424287Writing region info on filesystem at 1731640424287Initializing all the Stores at 1731640424289 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731640424289Cleaning up temporary data from old regions at 1731640424305 (+16 ms)Running coprocessor post-open hooks at 1731640424314 (+9 ms)Region opened successfully at 1731640424315 (+1 ms) 2024-11-15T03:13:44,315 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/d0eff1b91bc9b77579a5c44f1c985985 2024-11-15T03:13:44,316 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1731640423069.f5219cfe492c74eb8312930d328964b6., pid=48, masterSystemTime=1731640424199 2024-11-15T03:13:44,316 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for d0eff1b91bc9b77579a5c44f1c985985 2024-11-15T03:13:44,316 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for d0eff1b91bc9b77579a5c44f1c985985 2024-11-15T03:13:44,319 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1731640423069.c4b611da4a73d4cfc99379713486e21c. 2024-11-15T03:13:44,319 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened test,fff,1731640423069.c4b611da4a73d4cfc99379713486e21c. 2024-11-15T03:13:44,319 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(132): Open test,jjj,1731640423069.b1704275babe02e36b29cfaadf5b81af. 2024-11-15T03:13:44,319 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7752): Opening region: {ENCODED => b1704275babe02e36b29cfaadf5b81af, NAME => 'test,jjj,1731640423069.b1704275babe02e36b29cfaadf5b81af.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-15T03:13:44,320 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test b1704275babe02e36b29cfaadf5b81af 2024-11-15T03:13:44,320 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(898): Instantiated test,jjj,1731640423069.b1704275babe02e36b29cfaadf5b81af.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T03:13:44,320 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7794): checking encryption for b1704275babe02e36b29cfaadf5b81af 2024-11-15T03:13:44,320 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7797): checking classloading for b1704275babe02e36b29cfaadf5b81af 2024-11-15T03:13:44,320 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for d0eff1b91bc9b77579a5c44f1c985985 2024-11-15T03:13:44,321 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2266): Finished post open deploy task for test,iii,1731640423069.f5219cfe492c74eb8312930d328964b6. 2024-11-15T03:13:44,321 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(153): Opened test,iii,1731640423069.f5219cfe492c74eb8312930d328964b6. 2024-11-15T03:13:44,321 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(132): Open test,lll,1731640423069.860df8edb65f2fdd3c8bc00f89f94ba0. 2024-11-15T03:13:44,322 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7752): Opening region: {ENCODED => 860df8edb65f2fdd3c8bc00f89f94ba0, NAME => 'test,lll,1731640423069.860df8edb65f2fdd3c8bc00f89f94ba0.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-15T03:13:44,322 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 860df8edb65f2fdd3c8bc00f89f94ba0 2024-11-15T03:13:44,322 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(898): Instantiated test,lll,1731640423069.860df8edb65f2fdd3c8bc00f89f94ba0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T03:13:44,322 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7794): checking encryption for 860df8edb65f2fdd3c8bc00f89f94ba0 2024-11-15T03:13:44,322 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7797): checking classloading for 860df8edb65f2fdd3c8bc00f89f94ba0 2024-11-15T03:13:44,323 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42051 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=91f82191efb8,42027,1731640420387, table=test, region=a73d2d096e2f14a7e9365ac762927d55. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T03:13:44,325 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=24 2024-11-15T03:13:44,325 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure 21df709a703798d68290a0a63322957f, server=91f82191efb8,37523,1731640420593 in 186 msec 2024-11-15T03:13:44,327 INFO [StoreOpener-b1704275babe02e36b29cfaadf5b81af-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region b1704275babe02e36b29cfaadf5b81af 2024-11-15T03:13:44,328 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/d0eff1b91bc9b77579a5c44f1c985985/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T03:13:44,328 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened d0eff1b91bc9b77579a5c44f1c985985; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64968912, jitterRate=-0.03188776969909668}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T03:13:44,329 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d0eff1b91bc9b77579a5c44f1c985985 2024-11-15T03:13:44,329 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for d0eff1b91bc9b77579a5c44f1c985985: Running coprocessor pre-open hook at 1731640424303Writing region info on filesystem at 1731640424303Initializing all the Stores at 1731640424304 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731640424304Cleaning up temporary data from old regions at 1731640424316 (+12 ms)Running coprocessor post-open hooks at 1731640424329 (+13 ms)Region opened successfully at 1731640424329 2024-11-15T03:13:44,329 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=21df709a703798d68290a0a63322957f, ASSIGN in 514 msec 2024-11-15T03:13:44,329 INFO [StoreOpener-b1704275babe02e36b29cfaadf5b81af-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b1704275babe02e36b29cfaadf5b81af columnFamilyName family 2024-11-15T03:13:44,329 DEBUG [StoreOpener-b1704275babe02e36b29cfaadf5b81af-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T03:13:44,328 INFO [StoreOpener-860df8edb65f2fdd3c8bc00f89f94ba0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 860df8edb65f2fdd3c8bc00f89f94ba0 2024-11-15T03:13:44,330 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ggg,1731640423069.d0eff1b91bc9b77579a5c44f1c985985., pid=37, masterSystemTime=1731640424140 2024-11-15T03:13:44,330 INFO [StoreOpener-b1704275babe02e36b29cfaadf5b81af-1 {}] regionserver.HStore(327): Store=b1704275babe02e36b29cfaadf5b81af/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T03:13:44,331 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1038): replaying wal for b1704275babe02e36b29cfaadf5b81af 2024-11-15T03:13:44,331 INFO [StoreOpener-860df8edb65f2fdd3c8bc00f89f94ba0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 860df8edb65f2fdd3c8bc00f89f94ba0 columnFamilyName family 2024-11-15T03:13:44,331 DEBUG [StoreOpener-860df8edb65f2fdd3c8bc00f89f94ba0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T03:13:44,332 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=f5219cfe492c74eb8312930d328964b6, regionState=OPEN, openSeqNum=2, regionLocation=91f82191efb8,41451,1731640420638 2024-11-15T03:13:44,332 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/b1704275babe02e36b29cfaadf5b81af 2024-11-15T03:13:44,332 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/b1704275babe02e36b29cfaadf5b81af 2024-11-15T03:13:44,333 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1048): stopping wal replay for b1704275babe02e36b29cfaadf5b81af 2024-11-15T03:13:44,333 INFO [StoreOpener-860df8edb65f2fdd3c8bc00f89f94ba0-1 {}] regionserver.HStore(327): Store=860df8edb65f2fdd3c8bc00f89f94ba0/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T03:13:44,333 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1060): Cleaning up temporary data for b1704275babe02e36b29cfaadf5b81af 2024-11-15T03:13:44,333 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1038): replaying wal for 860df8edb65f2fdd3c8bc00f89f94ba0 2024-11-15T03:13:44,335 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ggg,1731640423069.d0eff1b91bc9b77579a5c44f1c985985. 2024-11-15T03:13:44,335 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/860df8edb65f2fdd3c8bc00f89f94ba0 2024-11-15T03:13:44,336 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened test,ggg,1731640423069.d0eff1b91bc9b77579a5c44f1c985985. 2024-11-15T03:13:44,336 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open test,vvv,1731640423069.dc93af816bdf94cf920a5adc31356783. 2024-11-15T03:13:44,336 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => dc93af816bdf94cf920a5adc31356783, NAME => 'test,vvv,1731640423069.dc93af816bdf94cf920a5adc31356783.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-15T03:13:44,336 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=48, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5219cfe492c74eb8312930d328964b6, server=91f82191efb8,41451,1731640420638 because future has completed 2024-11-15T03:13:44,336 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test dc93af816bdf94cf920a5adc31356783 2024-11-15T03:13:44,337 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated test,vvv,1731640423069.dc93af816bdf94cf920a5adc31356783.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T03:13:44,337 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for dc93af816bdf94cf920a5adc31356783 2024-11-15T03:13:44,337 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for dc93af816bdf94cf920a5adc31356783 2024-11-15T03:13:44,337 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=d0eff1b91bc9b77579a5c44f1c985985, regionState=OPEN, openSeqNum=2, regionLocation=91f82191efb8,42027,1731640420387 2024-11-15T03:13:44,337 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/860df8edb65f2fdd3c8bc00f89f94ba0 2024-11-15T03:13:44,337 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1093): writing seq id for b1704275babe02e36b29cfaadf5b81af 2024-11-15T03:13:44,338 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1048): stopping wal replay for 860df8edb65f2fdd3c8bc00f89f94ba0 2024-11-15T03:13:44,338 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1060): Cleaning up temporary data for 860df8edb65f2fdd3c8bc00f89f94ba0 2024-11-15T03:13:44,342 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure d0eff1b91bc9b77579a5c44f1c985985, server=91f82191efb8,42027,1731640420387 because future has completed 2024-11-15T03:13:44,343 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1093): writing seq id for 860df8edb65f2fdd3c8bc00f89f94ba0 2024-11-15T03:13:44,345 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/b1704275babe02e36b29cfaadf5b81af/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T03:13:44,346 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1114): Opened b1704275babe02e36b29cfaadf5b81af; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73746437, jitterRate=0.09890754520893097}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T03:13:44,347 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b1704275babe02e36b29cfaadf5b81af 2024-11-15T03:13:44,347 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1006): Region open journal for b1704275babe02e36b29cfaadf5b81af: Running coprocessor pre-open hook at 1731640424320Writing region info on filesystem at 1731640424320Initializing all the Stores at 1731640424322 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731640424322Cleaning up temporary data from old regions at 1731640424333 (+11 ms)Running coprocessor post-open hooks at 1731640424347 (+14 ms)Region opened successfully at 1731640424347 2024-11-15T03:13:44,348 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2236): Post open deploy tasks for test,jjj,1731640423069.b1704275babe02e36b29cfaadf5b81af., pid=50, masterSystemTime=1731640424234 2024-11-15T03:13:44,350 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=11 2024-11-15T03:13:44,350 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure d0eff1b91bc9b77579a5c44f1c985985, server=91f82191efb8,42027,1731640420387 in 312 msec 2024-11-15T03:13:44,352 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2266): Finished post open deploy task for test,jjj,1731640423069.b1704275babe02e36b29cfaadf5b81af. 2024-11-15T03:13:44,352 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(153): Opened test,jjj,1731640423069.b1704275babe02e36b29cfaadf5b81af. 2024-11-15T03:13:44,352 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open test,ppp,1731640423069.fac2a1b464a5ff2d581af6baef327bde. 2024-11-15T03:13:44,352 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => fac2a1b464a5ff2d581af6baef327bde, NAME => 'test,ppp,1731640423069.fac2a1b464a5ff2d581af6baef327bde.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-15T03:13:44,353 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test fac2a1b464a5ff2d581af6baef327bde 2024-11-15T03:13:44,353 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated test,ppp,1731640423069.fac2a1b464a5ff2d581af6baef327bde.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T03:13:44,353 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for fac2a1b464a5ff2d581af6baef327bde 2024-11-15T03:13:44,353 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for fac2a1b464a5ff2d581af6baef327bde 2024-11-15T03:13:44,354 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d0eff1b91bc9b77579a5c44f1c985985, ASSIGN in 539 msec 2024-11-15T03:13:44,356 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=b1704275babe02e36b29cfaadf5b81af, regionState=OPEN, openSeqNum=2, regionLocation=91f82191efb8,37523,1731640420593 2024-11-15T03:13:44,358 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/860df8edb65f2fdd3c8bc00f89f94ba0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T03:13:44,359 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure b1704275babe02e36b29cfaadf5b81af, server=91f82191efb8,37523,1731640420593 because future has completed 2024-11-15T03:13:44,360 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1114): Opened 860df8edb65f2fdd3c8bc00f89f94ba0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66896581, jitterRate=-0.003163263201713562}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T03:13:44,360 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 860df8edb65f2fdd3c8bc00f89f94ba0 2024-11-15T03:13:44,360 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1006): Region open journal for 860df8edb65f2fdd3c8bc00f89f94ba0: Running coprocessor pre-open hook at 1731640424322Writing region info on filesystem at 1731640424322Initializing all the Stores at 1731640424327 (+5 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731640424327Cleaning up temporary data from old regions at 1731640424338 (+11 ms)Running coprocessor post-open hooks at 1731640424360 (+22 ms)Region opened successfully at 1731640424360 2024-11-15T03:13:44,361 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1731640423069.860df8edb65f2fdd3c8bc00f89f94ba0., pid=44, masterSystemTime=1731640424199 2024-11-15T03:13:44,362 INFO [StoreOpener-fac2a1b464a5ff2d581af6baef327bde-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region fac2a1b464a5ff2d581af6baef327bde 2024-11-15T03:13:44,365 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=13 2024-11-15T03:13:44,365 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure f5219cfe492c74eb8312930d328964b6, server=91f82191efb8,41451,1731640420638 in 263 msec 2024-11-15T03:13:44,365 INFO [StoreOpener-fac2a1b464a5ff2d581af6baef327bde-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fac2a1b464a5ff2d581af6baef327bde columnFamilyName family 2024-11-15T03:13:44,365 DEBUG [StoreOpener-fac2a1b464a5ff2d581af6baef327bde-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T03:13:44,366 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1731640423069.860df8edb65f2fdd3c8bc00f89f94ba0. 2024-11-15T03:13:44,366 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(153): Opened test,lll,1731640423069.860df8edb65f2fdd3c8bc00f89f94ba0. 2024-11-15T03:13:44,367 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open test,,1731640423069.5965b596ce7b87fabde47569d60c3f95. 2024-11-15T03:13:44,367 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=860df8edb65f2fdd3c8bc00f89f94ba0, regionState=OPEN, openSeqNum=2, regionLocation=91f82191efb8,41451,1731640420638 2024-11-15T03:13:44,367 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => 5965b596ce7b87fabde47569d60c3f95, NAME => 'test,,1731640423069.5965b596ce7b87fabde47569d60c3f95.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-15T03:13:44,367 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 5965b596ce7b87fabde47569d60c3f95 2024-11-15T03:13:44,367 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated test,,1731640423069.5965b596ce7b87fabde47569d60c3f95.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T03:13:44,368 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for 5965b596ce7b87fabde47569d60c3f95 2024-11-15T03:13:44,368 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for 5965b596ce7b87fabde47569d60c3f95 2024-11-15T03:13:44,368 INFO [StoreOpener-fac2a1b464a5ff2d581af6baef327bde-1 {}] regionserver.HStore(327): Store=fac2a1b464a5ff2d581af6baef327bde/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T03:13:44,369 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for fac2a1b464a5ff2d581af6baef327bde 2024-11-15T03:13:44,370 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/fac2a1b464a5ff2d581af6baef327bde 2024-11-15T03:13:44,371 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/fac2a1b464a5ff2d581af6baef327bde 2024-11-15T03:13:44,371 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 860df8edb65f2fdd3c8bc00f89f94ba0, server=91f82191efb8,41451,1731640420638 because future has completed 2024-11-15T03:13:44,372 INFO [StoreOpener-dc93af816bdf94cf920a5adc31356783-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region dc93af816bdf94cf920a5adc31356783 2024-11-15T03:13:44,372 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for fac2a1b464a5ff2d581af6baef327bde 2024-11-15T03:13:44,372 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for fac2a1b464a5ff2d581af6baef327bde 2024-11-15T03:13:44,374 INFO [StoreOpener-dc93af816bdf94cf920a5adc31356783-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dc93af816bdf94cf920a5adc31356783 columnFamilyName family 2024-11-15T03:13:44,374 DEBUG [StoreOpener-dc93af816bdf94cf920a5adc31356783-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T03:13:44,375 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f5219cfe492c74eb8312930d328964b6, ASSIGN in 559 msec 2024-11-15T03:13:44,375 INFO [StoreOpener-dc93af816bdf94cf920a5adc31356783-1 {}] regionserver.HStore(327): Store=dc93af816bdf94cf920a5adc31356783/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T03:13:44,375 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for dc93af816bdf94cf920a5adc31356783 2024-11-15T03:13:44,376 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/dc93af816bdf94cf920a5adc31356783 2024-11-15T03:13:44,377 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for fac2a1b464a5ff2d581af6baef327bde 2024-11-15T03:13:44,377 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/dc93af816bdf94cf920a5adc31356783 2024-11-15T03:13:44,378 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for dc93af816bdf94cf920a5adc31356783 2024-11-15T03:13:44,378 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for dc93af816bdf94cf920a5adc31356783 2024-11-15T03:13:44,381 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for dc93af816bdf94cf920a5adc31356783 2024-11-15T03:13:44,383 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=16 2024-11-15T03:13:44,383 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 860df8edb65f2fdd3c8bc00f89f94ba0, server=91f82191efb8,41451,1731640420638 in 302 msec 2024-11-15T03:13:44,386 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=860df8edb65f2fdd3c8bc00f89f94ba0, ASSIGN in 573 msec 2024-11-15T03:13:44,392 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=14 2024-11-15T03:13:44,392 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure b1704275babe02e36b29cfaadf5b81af, server=91f82191efb8,37523,1731640420593 in 275 msec 2024-11-15T03:13:44,392 INFO [StoreOpener-5965b596ce7b87fabde47569d60c3f95-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 5965b596ce7b87fabde47569d60c3f95 2024-11-15T03:13:44,395 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/dc93af816bdf94cf920a5adc31356783/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T03:13:44,397 INFO [StoreOpener-5965b596ce7b87fabde47569d60c3f95-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5965b596ce7b87fabde47569d60c3f95 columnFamilyName family 2024-11-15T03:13:44,396 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened dc93af816bdf94cf920a5adc31356783; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61660955, jitterRate=-0.08118017017841339}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T03:13:44,397 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for dc93af816bdf94cf920a5adc31356783 2024-11-15T03:13:44,397 DEBUG [StoreOpener-5965b596ce7b87fabde47569d60c3f95-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T03:13:44,397 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for dc93af816bdf94cf920a5adc31356783: Running coprocessor pre-open hook at 1731640424337Writing region info on filesystem at 1731640424337Initializing all the Stores at 1731640424351 (+14 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731640424351Cleaning up temporary data from old regions at 1731640424378 (+27 ms)Running coprocessor post-open hooks at 1731640424397 (+19 ms)Region opened successfully at 1731640424397 2024-11-15T03:13:44,397 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=b1704275babe02e36b29cfaadf5b81af, ASSIGN in 581 msec 2024-11-15T03:13:44,398 INFO [StoreOpener-5965b596ce7b87fabde47569d60c3f95-1 {}] regionserver.HStore(327): Store=5965b596ce7b87fabde47569d60c3f95/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T03:13:44,398 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for 5965b596ce7b87fabde47569d60c3f95 2024-11-15T03:13:44,398 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1731640423069.dc93af816bdf94cf920a5adc31356783., pid=36, masterSystemTime=1731640424140 2024-11-15T03:13:44,399 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/5965b596ce7b87fabde47569d60c3f95 2024-11-15T03:13:44,400 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=c4b611da4a73d4cfc99379713486e21c, regionState=OPEN, openSeqNum=2, regionLocation=91f82191efb8,37523,1731640420593 2024-11-15T03:13:44,401 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/fac2a1b464a5ff2d581af6baef327bde/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T03:13:44,402 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened fac2a1b464a5ff2d581af6baef327bde; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67562170, jitterRate=0.006754785776138306}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T03:13:44,402 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fac2a1b464a5ff2d581af6baef327bde 2024-11-15T03:13:44,402 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for fac2a1b464a5ff2d581af6baef327bde: Running coprocessor pre-open hook at 1731640424353Writing region info on filesystem at 1731640424353Initializing all the Stores at 1731640424355 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731640424355Cleaning up temporary data from old regions at 1731640424372 (+17 ms)Running coprocessor post-open hooks at 1731640424402 (+30 ms)Region opened successfully at 1731640424402 2024-11-15T03:13:44,401 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/5965b596ce7b87fabde47569d60c3f95 2024-11-15T03:13:44,404 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1731640423069.dc93af816bdf94cf920a5adc31356783. 2024-11-15T03:13:44,404 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened test,vvv,1731640423069.dc93af816bdf94cf920a5adc31356783. 2024-11-15T03:13:44,404 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1731640423069.fac2a1b464a5ff2d581af6baef327bde., pid=53, masterSystemTime=1731640424234 2024-11-15T03:13:44,404 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open test,ccc,1731640423069.5971f307349447e7f24b0dd05b96cffc. 2024-11-15T03:13:44,404 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => 5971f307349447e7f24b0dd05b96cffc, NAME => 'test,ccc,1731640423069.5971f307349447e7f24b0dd05b96cffc.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-15T03:13:44,404 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure c4b611da4a73d4cfc99379713486e21c, server=91f82191efb8,37523,1731640420593 because future has completed 2024-11-15T03:13:44,405 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 5971f307349447e7f24b0dd05b96cffc 2024-11-15T03:13:44,405 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated test,ccc,1731640423069.5971f307349447e7f24b0dd05b96cffc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T03:13:44,405 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for 5971f307349447e7f24b0dd05b96cffc 2024-11-15T03:13:44,405 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for 5971f307349447e7f24b0dd05b96cffc 2024-11-15T03:13:44,405 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for 5965b596ce7b87fabde47569d60c3f95 2024-11-15T03:13:44,405 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for 5965b596ce7b87fabde47569d60c3f95 2024-11-15T03:13:44,407 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=dc93af816bdf94cf920a5adc31356783, regionState=OPEN, openSeqNum=2, regionLocation=91f82191efb8,42027,1731640420387 2024-11-15T03:13:44,407 INFO [StoreOpener-5971f307349447e7f24b0dd05b96cffc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 5971f307349447e7f24b0dd05b96cffc 2024-11-15T03:13:44,409 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ppp,1731640423069.fac2a1b464a5ff2d581af6baef327bde. 2024-11-15T03:13:44,409 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened test,ppp,1731640423069.fac2a1b464a5ff2d581af6baef327bde. 2024-11-15T03:13:44,409 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(132): Open test,www,1731640423069.155ef591549501ad2da24d2c559fb7f9. 2024-11-15T03:13:44,409 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7752): Opening region: {ENCODED => 155ef591549501ad2da24d2c559fb7f9, NAME => 'test,www,1731640423069.155ef591549501ad2da24d2c559fb7f9.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-11-15T03:13:44,410 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 155ef591549501ad2da24d2c559fb7f9 2024-11-15T03:13:44,410 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(898): Instantiated test,www,1731640423069.155ef591549501ad2da24d2c559fb7f9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T03:13:44,410 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7794): checking encryption for 155ef591549501ad2da24d2c559fb7f9 2024-11-15T03:13:44,410 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7797): checking classloading for 155ef591549501ad2da24d2c559fb7f9 2024-11-15T03:13:44,412 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure dc93af816bdf94cf920a5adc31356783, server=91f82191efb8,42027,1731640420387 because future has completed 2024-11-15T03:13:44,412 INFO [StoreOpener-5971f307349447e7f24b0dd05b96cffc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5971f307349447e7f24b0dd05b96cffc columnFamilyName family 2024-11-15T03:13:44,412 DEBUG [StoreOpener-5971f307349447e7f24b0dd05b96cffc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T03:13:44,413 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for 5965b596ce7b87fabde47569d60c3f95 2024-11-15T03:13:44,413 INFO [StoreOpener-5971f307349447e7f24b0dd05b96cffc-1 {}] regionserver.HStore(327): Store=5971f307349447e7f24b0dd05b96cffc/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T03:13:44,414 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for 5971f307349447e7f24b0dd05b96cffc 2024-11-15T03:13:44,414 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=fac2a1b464a5ff2d581af6baef327bde, regionState=OPEN, openSeqNum=2, regionLocation=91f82191efb8,37523,1731640420593 2024-11-15T03:13:44,415 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/5971f307349447e7f24b0dd05b96cffc 2024-11-15T03:13:44,416 INFO [StoreOpener-155ef591549501ad2da24d2c559fb7f9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 155ef591549501ad2da24d2c559fb7f9 2024-11-15T03:13:44,416 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/5971f307349447e7f24b0dd05b96cffc 2024-11-15T03:13:44,417 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for 5971f307349447e7f24b0dd05b96cffc 2024-11-15T03:13:44,417 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for 5971f307349447e7f24b0dd05b96cffc 2024-11-15T03:13:44,418 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure fac2a1b464a5ff2d581af6baef327bde, server=91f82191efb8,37523,1731640420593 because future has completed 2024-11-15T03:13:44,420 INFO [StoreOpener-155ef591549501ad2da24d2c559fb7f9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 155ef591549501ad2da24d2c559fb7f9 columnFamilyName family 2024-11-15T03:13:44,420 DEBUG [StoreOpener-155ef591549501ad2da24d2c559fb7f9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T03:13:44,422 INFO [StoreOpener-155ef591549501ad2da24d2c559fb7f9-1 {}] regionserver.HStore(327): Store=155ef591549501ad2da24d2c559fb7f9/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T03:13:44,422 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1038): replaying wal for 155ef591549501ad2da24d2c559fb7f9 2024-11-15T03:13:44,424 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/155ef591549501ad2da24d2c559fb7f9 2024-11-15T03:13:44,425 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/155ef591549501ad2da24d2c559fb7f9 2024-11-15T03:13:44,425 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=26 2024-11-15T03:13:44,426 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure dc93af816bdf94cf920a5adc31356783, server=91f82191efb8,42027,1731640420387 in 398 msec 2024-11-15T03:13:44,426 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1048): stopping wal replay for 155ef591549501ad2da24d2c559fb7f9 2024-11-15T03:13:44,426 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/5965b596ce7b87fabde47569d60c3f95/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T03:13:44,426 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1060): Cleaning up temporary data for 155ef591549501ad2da24d2c559fb7f9 2024-11-15T03:13:44,427 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened 5965b596ce7b87fabde47569d60c3f95; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70589068, jitterRate=0.05185908079147339}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T03:13:44,427 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5965b596ce7b87fabde47569d60c3f95 2024-11-15T03:13:44,428 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for 5965b596ce7b87fabde47569d60c3f95: Running coprocessor pre-open hook at 1731640424368Writing region info on filesystem at 1731640424368Initializing all the Stores at 1731640424372 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731640424372Cleaning up temporary data from old regions at 1731640424405 (+33 ms)Running coprocessor post-open hooks at 1731640424427 (+22 ms)Region opened successfully at 1731640424428 (+1 ms) 2024-11-15T03:13:44,429 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): writing seq id for 155ef591549501ad2da24d2c559fb7f9 2024-11-15T03:13:44,430 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=dc93af816bdf94cf920a5adc31356783, ASSIGN in 615 msec 2024-11-15T03:13:44,432 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1731640423069.5965b596ce7b87fabde47569d60c3f95., pid=46, masterSystemTime=1731640424199 2024-11-15T03:13:44,430 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for 5971f307349447e7f24b0dd05b96cffc 2024-11-15T03:13:44,436 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42051 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=91f82191efb8,37523,1731640420593, table=test, region=c4b611da4a73d4cfc99379713486e21c. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T03:13:44,437 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42051 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=91f82191efb8,42027,1731640420387, table=test, region=a73d2d096e2f14a7e9365ac762927d55. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T03:13:44,437 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/155ef591549501ad2da24d2c559fb7f9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T03:13:44,435 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for test,,1731640423069.5965b596ce7b87fabde47569d60c3f95. 2024-11-15T03:13:44,437 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened test,,1731640423069.5965b596ce7b87fabde47569d60c3f95. 2024-11-15T03:13:44,437 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open test,sss,1731640423069.4ef03d91ad777a77cf66f510e28419ba. 2024-11-15T03:13:44,437 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => 4ef03d91ad777a77cf66f510e28419ba, NAME => 'test,sss,1731640423069.4ef03d91ad777a77cf66f510e28419ba.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-15T03:13:44,438 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4ef03d91ad777a77cf66f510e28419ba 2024-11-15T03:13:44,438 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1114): Opened 155ef591549501ad2da24d2c559fb7f9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71518572, jitterRate=0.06570976972579956}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T03:13:44,438 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 155ef591549501ad2da24d2c559fb7f9 2024-11-15T03:13:44,438 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated test,sss,1731640423069.4ef03d91ad777a77cf66f510e28419ba.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T03:13:44,438 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for 4ef03d91ad777a77cf66f510e28419ba 2024-11-15T03:13:44,438 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1006): Region open journal for 155ef591549501ad2da24d2c559fb7f9: Running coprocessor pre-open hook at 1731640424410Writing region info on filesystem at 1731640424410Initializing all the Stores at 1731640424412 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731640424412Cleaning up temporary data from old regions at 1731640424426 (+14 ms)Running coprocessor post-open hooks at 1731640424438 (+12 ms)Region opened successfully at 1731640424438 2024-11-15T03:13:44,438 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for 4ef03d91ad777a77cf66f510e28419ba 2024-11-15T03:13:44,440 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=20 2024-11-15T03:13:44,440 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure fac2a1b464a5ff2d581af6baef327bde, server=91f82191efb8,37523,1731640420593 in 306 msec 2024-11-15T03:13:44,441 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2236): Post open deploy tasks for test,www,1731640423069.155ef591549501ad2da24d2c559fb7f9., pid=49, masterSystemTime=1731640424234 2024-11-15T03:13:44,444 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fac2a1b464a5ff2d581af6baef327bde, ASSIGN in 629 msec 2024-11-15T03:13:44,444 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2266): Finished post open deploy task for test,www,1731640423069.155ef591549501ad2da24d2c559fb7f9. 2024-11-15T03:13:44,444 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(153): Opened test,www,1731640423069.155ef591549501ad2da24d2c559fb7f9. 2024-11-15T03:13:44,445 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open test,zzz,1731640423069.17e8b3b889a606b8c6826552e784d401. 2024-11-15T03:13:44,445 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => 17e8b3b889a606b8c6826552e784d401, NAME => 'test,zzz,1731640423069.17e8b3b889a606b8c6826552e784d401.', STARTKEY => 'zzz', ENDKEY => ''} 2024-11-15T03:13:44,446 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 17e8b3b889a606b8c6826552e784d401 2024-11-15T03:13:44,446 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=155ef591549501ad2da24d2c559fb7f9, regionState=OPEN, openSeqNum=2, regionLocation=91f82191efb8,37523,1731640420593 2024-11-15T03:13:44,446 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated test,zzz,1731640423069.17e8b3b889a606b8c6826552e784d401.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T03:13:44,446 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for 17e8b3b889a606b8c6826552e784d401 2024-11-15T03:13:44,446 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for 17e8b3b889a606b8c6826552e784d401 2024-11-15T03:13:44,449 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 155ef591549501ad2da24d2c559fb7f9, server=91f82191efb8,37523,1731640420593 because future has completed 2024-11-15T03:13:44,451 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/5971f307349447e7f24b0dd05b96cffc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T03:13:44,451 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=5965b596ce7b87fabde47569d60c3f95, regionState=OPEN, openSeqNum=2, regionLocation=91f82191efb8,41451,1731640420638 2024-11-15T03:13:44,452 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened 5971f307349447e7f24b0dd05b96cffc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64011670, jitterRate=-0.0461517870426178}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T03:13:44,452 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5971f307349447e7f24b0dd05b96cffc 2024-11-15T03:13:44,452 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for 5971f307349447e7f24b0dd05b96cffc: Running coprocessor pre-open hook at 1731640424405Writing region info on filesystem at 1731640424405Initializing all the Stores at 1731640424406 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731640424406Cleaning up temporary data from old regions at 1731640424417 (+11 ms)Running coprocessor post-open hooks at 1731640424452 (+35 ms)Region opened successfully at 1731640424452 2024-11-15T03:13:44,454 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1731640423069.5971f307349447e7f24b0dd05b96cffc., pid=38, masterSystemTime=1731640424140 2024-11-15T03:13:44,454 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5965b596ce7b87fabde47569d60c3f95, server=91f82191efb8,41451,1731640420638 because future has completed 2024-11-15T03:13:44,455 INFO [StoreOpener-4ef03d91ad777a77cf66f510e28419ba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4ef03d91ad777a77cf66f510e28419ba 2024-11-15T03:13:44,455 INFO [StoreOpener-17e8b3b889a606b8c6826552e784d401-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 17e8b3b889a606b8c6826552e784d401 2024-11-15T03:13:44,457 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ccc,1731640423069.5971f307349447e7f24b0dd05b96cffc. 2024-11-15T03:13:44,458 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened test,ccc,1731640423069.5971f307349447e7f24b0dd05b96cffc. 2024-11-15T03:13:44,458 INFO [StoreOpener-4ef03d91ad777a77cf66f510e28419ba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4ef03d91ad777a77cf66f510e28419ba columnFamilyName family 2024-11-15T03:13:44,458 DEBUG [StoreOpener-4ef03d91ad777a77cf66f510e28419ba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T03:13:44,458 INFO [StoreOpener-17e8b3b889a606b8c6826552e784d401-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 17e8b3b889a606b8c6826552e784d401 columnFamilyName family 2024-11-15T03:13:44,459 INFO [StoreOpener-4ef03d91ad777a77cf66f510e28419ba-1 {}] regionserver.HStore(327): Store=4ef03d91ad777a77cf66f510e28419ba/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T03:13:44,460 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for 4ef03d91ad777a77cf66f510e28419ba 2024-11-15T03:13:44,461 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/4ef03d91ad777a77cf66f510e28419ba 2024-11-15T03:13:44,461 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=27 2024-11-15T03:13:44,461 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure 155ef591549501ad2da24d2c559fb7f9, server=91f82191efb8,37523,1731640420593 in 351 msec 2024-11-15T03:13:44,462 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/4ef03d91ad777a77cf66f510e28419ba 2024-11-15T03:13:44,463 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for 4ef03d91ad777a77cf66f510e28419ba 2024-11-15T03:13:44,463 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for 4ef03d91ad777a77cf66f510e28419ba 2024-11-15T03:13:44,464 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=155ef591549501ad2da24d2c559fb7f9, ASSIGN in 650 msec 2024-11-15T03:13:44,466 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=5971f307349447e7f24b0dd05b96cffc, regionState=OPEN, openSeqNum=2, regionLocation=91f82191efb8,42027,1731640420387 2024-11-15T03:13:44,467 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for 4ef03d91ad777a77cf66f510e28419ba 2024-11-15T03:13:44,469 DEBUG [StoreOpener-17e8b3b889a606b8c6826552e784d401-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T03:13:44,469 INFO [StoreOpener-17e8b3b889a606b8c6826552e784d401-1 {}] regionserver.HStore(327): Store=17e8b3b889a606b8c6826552e784d401/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T03:13:44,471 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5971f307349447e7f24b0dd05b96cffc, server=91f82191efb8,42027,1731640420387 because future has completed 2024-11-15T03:13:44,471 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for 17e8b3b889a606b8c6826552e784d401 2024-11-15T03:13:44,471 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/4ef03d91ad777a77cf66f510e28419ba/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T03:13:44,472 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened 4ef03d91ad777a77cf66f510e28419ba; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65556144, jitterRate=-0.023137331008911133}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T03:13:44,472 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4ef03d91ad777a77cf66f510e28419ba 2024-11-15T03:13:44,472 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for 4ef03d91ad777a77cf66f510e28419ba: Running coprocessor pre-open hook at 1731640424438Writing region info on filesystem at 1731640424438Initializing all the Stores at 1731640424440 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731640424440Cleaning up temporary data from old regions at 1731640424463 (+23 ms)Running coprocessor post-open hooks at 1731640424472 (+9 ms)Region opened successfully at 1731640424472 2024-11-15T03:13:44,474 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/17e8b3b889a606b8c6826552e784d401 2024-11-15T03:13:44,474 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1731640423069.4ef03d91ad777a77cf66f510e28419ba., pid=41, masterSystemTime=1731640424199 2024-11-15T03:13:44,475 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/17e8b3b889a606b8c6826552e784d401 2024-11-15T03:13:44,476 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for 17e8b3b889a606b8c6826552e784d401 2024-11-15T03:13:44,476 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for 17e8b3b889a606b8c6826552e784d401 2024-11-15T03:13:44,477 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1731640423069.4ef03d91ad777a77cf66f510e28419ba. 2024-11-15T03:13:44,477 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened test,sss,1731640423069.4ef03d91ad777a77cf66f510e28419ba. 2024-11-15T03:13:44,478 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open test,rrr,1731640423069.dc930ecfca79abbf57ed28c20f79230f. 2024-11-15T03:13:44,478 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => dc930ecfca79abbf57ed28c20f79230f, NAME => 'test,rrr,1731640423069.dc930ecfca79abbf57ed28c20f79230f.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-15T03:13:44,478 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test dc930ecfca79abbf57ed28c20f79230f 2024-11-15T03:13:44,478 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated test,rrr,1731640423069.dc930ecfca79abbf57ed28c20f79230f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T03:13:44,479 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for dc930ecfca79abbf57ed28c20f79230f 2024-11-15T03:13:44,479 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for dc930ecfca79abbf57ed28c20f79230f 2024-11-15T03:13:44,479 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=7 2024-11-15T03:13:44,480 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 5971f307349447e7f24b0dd05b96cffc, server=91f82191efb8,42027,1731640420387 in 434 msec 2024-11-15T03:13:44,481 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for 17e8b3b889a606b8c6826552e784d401 2024-11-15T03:13:44,483 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN in 669 msec 2024-11-15T03:13:44,484 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=4ef03d91ad777a77cf66f510e28419ba, regionState=OPEN, openSeqNum=2, regionLocation=91f82191efb8,41451,1731640420638 2024-11-15T03:13:44,484 INFO [StoreOpener-dc930ecfca79abbf57ed28c20f79230f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region dc930ecfca79abbf57ed28c20f79230f 2024-11-15T03:13:44,485 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/17e8b3b889a606b8c6826552e784d401/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T03:13:44,486 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened 17e8b3b889a606b8c6826552e784d401; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66598292, jitterRate=-0.0076081156730651855}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T03:13:44,486 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 17e8b3b889a606b8c6826552e784d401 2024-11-15T03:13:44,486 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for 17e8b3b889a606b8c6826552e784d401: Running coprocessor pre-open hook at 1731640424446Writing region info on filesystem at 1731640424446Initializing all the Stores at 1731640424449 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731640424449Cleaning up temporary data from old regions at 1731640424476 (+27 ms)Running coprocessor post-open hooks at 1731640424486 (+10 ms)Region opened successfully at 1731640424486 2024-11-15T03:13:44,487 INFO [StoreOpener-dc930ecfca79abbf57ed28c20f79230f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dc930ecfca79abbf57ed28c20f79230f columnFamilyName family 2024-11-15T03:13:44,487 DEBUG [StoreOpener-dc930ecfca79abbf57ed28c20f79230f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T03:13:44,488 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for test,zzz,1731640423069.17e8b3b889a606b8c6826552e784d401., pid=45, masterSystemTime=1731640424234 2024-11-15T03:13:44,488 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4ef03d91ad777a77cf66f510e28419ba, server=91f82191efb8,41451,1731640420638 because future has completed 2024-11-15T03:13:44,488 INFO [StoreOpener-dc930ecfca79abbf57ed28c20f79230f-1 {}] regionserver.HStore(327): Store=dc930ecfca79abbf57ed28c20f79230f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T03:13:44,488 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for dc930ecfca79abbf57ed28c20f79230f 2024-11-15T03:13:44,490 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/dc930ecfca79abbf57ed28c20f79230f 2024-11-15T03:13:44,491 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/dc930ecfca79abbf57ed28c20f79230f 2024-11-15T03:13:44,491 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for dc930ecfca79abbf57ed28c20f79230f 2024-11-15T03:13:44,492 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for dc930ecfca79abbf57ed28c20f79230f 2024-11-15T03:13:44,492 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for test,zzz,1731640423069.17e8b3b889a606b8c6826552e784d401. 2024-11-15T03:13:44,492 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened test,zzz,1731640423069.17e8b3b889a606b8c6826552e784d401. 2024-11-15T03:13:44,493 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(132): Open test,bbb,1731640423069.6130297c45bd23340b368de2cc708c90. 2024-11-15T03:13:44,493 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7752): Opening region: {ENCODED => 6130297c45bd23340b368de2cc708c90, NAME => 'test,bbb,1731640423069.6130297c45bd23340b368de2cc708c90.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-11-15T03:13:44,493 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 6130297c45bd23340b368de2cc708c90 2024-11-15T03:13:44,493 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(898): Instantiated test,bbb,1731640423069.6130297c45bd23340b368de2cc708c90.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T03:13:44,494 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7794): checking encryption for 6130297c45bd23340b368de2cc708c90 2024-11-15T03:13:44,494 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7797): checking classloading for 6130297c45bd23340b368de2cc708c90 2024-11-15T03:13:44,495 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=5 2024-11-15T03:13:44,495 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 5965b596ce7b87fabde47569d60c3f95, server=91f82191efb8,41451,1731640420638 in 403 msec 2024-11-15T03:13:44,495 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for dc930ecfca79abbf57ed28c20f79230f 2024-11-15T03:13:44,499 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=5965b596ce7b87fabde47569d60c3f95, ASSIGN in 684 msec 2024-11-15T03:13:44,501 INFO [StoreOpener-6130297c45bd23340b368de2cc708c90-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 6130297c45bd23340b368de2cc708c90 2024-11-15T03:13:44,501 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=17e8b3b889a606b8c6826552e784d401, regionState=OPEN, openSeqNum=2, regionLocation=91f82191efb8,37523,1731640420593 2024-11-15T03:13:44,504 INFO [StoreOpener-6130297c45bd23340b368de2cc708c90-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6130297c45bd23340b368de2cc708c90 columnFamilyName family 2024-11-15T03:13:44,504 DEBUG [StoreOpener-6130297c45bd23340b368de2cc708c90-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T03:13:44,504 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 17e8b3b889a606b8c6826552e784d401, server=91f82191efb8,37523,1731640420593 because future has completed 2024-11-15T03:13:44,506 INFO [StoreOpener-6130297c45bd23340b368de2cc708c90-1 {}] regionserver.HStore(327): Store=6130297c45bd23340b368de2cc708c90/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T03:13:44,506 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1038): replaying wal for 6130297c45bd23340b368de2cc708c90 2024-11-15T03:13:44,508 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/6130297c45bd23340b368de2cc708c90 2024-11-15T03:13:44,509 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/6130297c45bd23340b368de2cc708c90 2024-11-15T03:13:44,510 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1048): stopping wal replay for 6130297c45bd23340b368de2cc708c90 2024-11-15T03:13:44,510 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1060): Cleaning up temporary data for 6130297c45bd23340b368de2cc708c90 2024-11-15T03:13:44,511 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=23 2024-11-15T03:13:44,511 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 4ef03d91ad777a77cf66f510e28419ba, server=91f82191efb8,41451,1731640420638 in 449 msec 2024-11-15T03:13:44,512 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/dc930ecfca79abbf57ed28c20f79230f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T03:13:44,513 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened dc930ecfca79abbf57ed28c20f79230f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65164258, jitterRate=-0.028976887464523315}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T03:13:44,514 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for dc930ecfca79abbf57ed28c20f79230f 2024-11-15T03:13:44,514 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for dc930ecfca79abbf57ed28c20f79230f: Running coprocessor pre-open hook at 1731640424479Writing region info on filesystem at 1731640424479Initializing all the Stores at 1731640424480 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731640424480Cleaning up temporary data from old regions at 1731640424492 (+12 ms)Running coprocessor post-open hooks at 1731640424514 (+22 ms)Region opened successfully at 1731640424514 2024-11-15T03:13:44,515 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1093): writing seq id for 6130297c45bd23340b368de2cc708c90 2024-11-15T03:13:44,515 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1731640423069.dc930ecfca79abbf57ed28c20f79230f., pid=39, masterSystemTime=1731640424199 2024-11-15T03:13:44,517 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=4ef03d91ad777a77cf66f510e28419ba, ASSIGN in 700 msec 2024-11-15T03:13:44,518 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1731640423069.dc930ecfca79abbf57ed28c20f79230f. 2024-11-15T03:13:44,518 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened test,rrr,1731640423069.dc930ecfca79abbf57ed28c20f79230f. 2024-11-15T03:13:44,518 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open test,eee,1731640423069.e0f9e1e28ca8906f0f82f959a60c55ce. 2024-11-15T03:13:44,518 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => e0f9e1e28ca8906f0f82f959a60c55ce, NAME => 'test,eee,1731640423069.e0f9e1e28ca8906f0f82f959a60c55ce.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-15T03:13:44,519 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test e0f9e1e28ca8906f0f82f959a60c55ce 2024-11-15T03:13:44,519 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated test,eee,1731640423069.e0f9e1e28ca8906f0f82f959a60c55ce.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T03:13:44,519 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for e0f9e1e28ca8906f0f82f959a60c55ce 2024-11-15T03:13:44,519 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for e0f9e1e28ca8906f0f82f959a60c55ce 2024-11-15T03:13:44,521 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/6130297c45bd23340b368de2cc708c90/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T03:13:44,523 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1114): Opened 6130297c45bd23340b368de2cc708c90; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61978713, jitterRate=-0.07644520699977875}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T03:13:44,523 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6130297c45bd23340b368de2cc708c90 2024-11-15T03:13:44,523 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1006): Region open journal for 6130297c45bd23340b368de2cc708c90: Running coprocessor pre-open hook at 1731640424494Writing region info on filesystem at 1731640424494Initializing all the Stores at 1731640424495 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731640424495Cleaning up temporary data from old regions at 1731640424510 (+15 ms)Running coprocessor post-open hooks at 1731640424523 (+13 ms)Region opened successfully at 1731640424523 2024-11-15T03:13:44,524 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2236): Post open deploy tasks for test,bbb,1731640423069.6130297c45bd23340b368de2cc708c90., pid=56, masterSystemTime=1731640424234 2024-11-15T03:13:44,525 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=30 2024-11-15T03:13:44,525 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure 17e8b3b889a606b8c6826552e784d401, server=91f82191efb8,37523,1731640420593 in 438 msec 2024-11-15T03:13:44,527 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2266): Finished post open deploy task for test,bbb,1731640423069.6130297c45bd23340b368de2cc708c90. 2024-11-15T03:13:44,527 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(153): Opened test,bbb,1731640423069.6130297c45bd23340b368de2cc708c90. 2024-11-15T03:13:44,528 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(132): Open test,mmm,1731640423069.eaf85cc654192e46d3ea0904493eea12. 2024-11-15T03:13:44,528 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=17e8b3b889a606b8c6826552e784d401, ASSIGN in 714 msec 2024-11-15T03:13:44,528 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7752): Opening region: {ENCODED => eaf85cc654192e46d3ea0904493eea12, NAME => 'test,mmm,1731640423069.eaf85cc654192e46d3ea0904493eea12.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-15T03:13:44,529 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test eaf85cc654192e46d3ea0904493eea12 2024-11-15T03:13:44,529 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(898): Instantiated test,mmm,1731640423069.eaf85cc654192e46d3ea0904493eea12.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T03:13:44,529 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7794): checking encryption for eaf85cc654192e46d3ea0904493eea12 2024-11-15T03:13:44,529 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7797): checking classloading for eaf85cc654192e46d3ea0904493eea12 2024-11-15T03:13:44,530 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=6130297c45bd23340b368de2cc708c90, regionState=OPEN, openSeqNum=2, regionLocation=91f82191efb8,37523,1731640420593 2024-11-15T03:13:44,532 INFO [StoreOpener-e0f9e1e28ca8906f0f82f959a60c55ce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region e0f9e1e28ca8906f0f82f959a60c55ce 2024-11-15T03:13:44,535 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6130297c45bd23340b368de2cc708c90, server=91f82191efb8,37523,1731640420593 because future has completed 2024-11-15T03:13:44,536 INFO [StoreOpener-e0f9e1e28ca8906f0f82f959a60c55ce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e0f9e1e28ca8906f0f82f959a60c55ce columnFamilyName family 2024-11-15T03:13:44,536 DEBUG [StoreOpener-e0f9e1e28ca8906f0f82f959a60c55ce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T03:13:44,537 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=dc930ecfca79abbf57ed28c20f79230f, regionState=OPEN, openSeqNum=2, regionLocation=91f82191efb8,41451,1731640420638 2024-11-15T03:13:44,537 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42051 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=91f82191efb8,41451,1731640420638, table=test, region=dc930ecfca79abbf57ed28c20f79230f. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T03:13:44,538 INFO [StoreOpener-e0f9e1e28ca8906f0f82f959a60c55ce-1 {}] regionserver.HStore(327): Store=e0f9e1e28ca8906f0f82f959a60c55ce/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T03:13:44,538 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for e0f9e1e28ca8906f0f82f959a60c55ce 2024-11-15T03:13:44,539 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/e0f9e1e28ca8906f0f82f959a60c55ce 2024-11-15T03:13:44,540 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/e0f9e1e28ca8906f0f82f959a60c55ce 2024-11-15T03:13:44,541 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for e0f9e1e28ca8906f0f82f959a60c55ce 2024-11-15T03:13:44,541 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for e0f9e1e28ca8906f0f82f959a60c55ce 2024-11-15T03:13:44,542 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure dc930ecfca79abbf57ed28c20f79230f, server=91f82191efb8,41451,1731640420638 because future has completed 2024-11-15T03:13:44,543 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42051 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=91f82191efb8,37523,1731640420593, table=test, region=c4b611da4a73d4cfc99379713486e21c. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T03:13:44,544 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42051 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=91f82191efb8,42027,1731640420387, table=test, region=a73d2d096e2f14a7e9365ac762927d55. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T03:13:44,546 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for e0f9e1e28ca8906f0f82f959a60c55ce 2024-11-15T03:13:44,548 INFO [StoreOpener-eaf85cc654192e46d3ea0904493eea12-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region eaf85cc654192e46d3ea0904493eea12 2024-11-15T03:13:44,549 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=6 2024-11-15T03:13:44,549 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure 6130297c45bd23340b368de2cc708c90, server=91f82191efb8,37523,1731640420593 in 400 msec 2024-11-15T03:13:44,550 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/e0f9e1e28ca8906f0f82f959a60c55ce/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T03:13:44,551 INFO [StoreOpener-eaf85cc654192e46d3ea0904493eea12-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eaf85cc654192e46d3ea0904493eea12 columnFamilyName family 2024-11-15T03:13:44,551 DEBUG [StoreOpener-eaf85cc654192e46d3ea0904493eea12-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T03:13:44,551 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened e0f9e1e28ca8906f0f82f959a60c55ce; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75068006, jitterRate=0.11860045790672302}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T03:13:44,551 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e0f9e1e28ca8906f0f82f959a60c55ce 2024-11-15T03:13:44,551 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for e0f9e1e28ca8906f0f82f959a60c55ce: Running coprocessor pre-open hook at 1731640424519Writing region info on filesystem at 1731640424519Initializing all the Stores at 1731640424521 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731640424521Cleaning up temporary data from old regions at 1731640424541 (+20 ms)Running coprocessor post-open hooks at 1731640424551 (+10 ms)Region opened successfully at 1731640424551 2024-11-15T03:13:44,552 INFO [StoreOpener-eaf85cc654192e46d3ea0904493eea12-1 {}] regionserver.HStore(327): Store=eaf85cc654192e46d3ea0904493eea12/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T03:13:44,552 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1038): replaying wal for eaf85cc654192e46d3ea0904493eea12 2024-11-15T03:13:44,553 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6130297c45bd23340b368de2cc708c90, ASSIGN in 738 msec 2024-11-15T03:13:44,553 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1731640423069.e0f9e1e28ca8906f0f82f959a60c55ce., pid=47, masterSystemTime=1731640424199 2024-11-15T03:13:44,553 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/eaf85cc654192e46d3ea0904493eea12 2024-11-15T03:13:44,556 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/eaf85cc654192e46d3ea0904493eea12 2024-11-15T03:13:44,556 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1731640423069.e0f9e1e28ca8906f0f82f959a60c55ce. 2024-11-15T03:13:44,556 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened test,eee,1731640423069.e0f9e1e28ca8906f0f82f959a60c55ce. 2024-11-15T03:13:44,556 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1048): stopping wal replay for eaf85cc654192e46d3ea0904493eea12 2024-11-15T03:13:44,557 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1060): Cleaning up temporary data for eaf85cc654192e46d3ea0904493eea12 2024-11-15T03:13:44,560 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): writing seq id for eaf85cc654192e46d3ea0904493eea12 2024-11-15T03:13:44,560 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=22 2024-11-15T03:13:44,560 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure dc930ecfca79abbf57ed28c20f79230f, server=91f82191efb8,41451,1731640420638 in 509 msec 2024-11-15T03:13:44,564 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=dc930ecfca79abbf57ed28c20f79230f, ASSIGN in 750 msec 2024-11-15T03:13:44,564 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/eaf85cc654192e46d3ea0904493eea12/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T03:13:44,565 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1114): Opened eaf85cc654192e46d3ea0904493eea12; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73802165, jitterRate=0.09973795711994171}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T03:13:44,565 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1122): Running coprocessor post-open hooks for eaf85cc654192e46d3ea0904493eea12 2024-11-15T03:13:44,566 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1006): Region open journal for eaf85cc654192e46d3ea0904493eea12: Running coprocessor pre-open hook at 1731640424529Writing region info on filesystem at 1731640424529Initializing all the Stores at 1731640424533 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731640424533Cleaning up temporary data from old regions at 1731640424557 (+24 ms)Running coprocessor post-open hooks at 1731640424565 (+8 ms)Region opened successfully at 1731640424565 2024-11-15T03:13:44,566 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=e0f9e1e28ca8906f0f82f959a60c55ce, regionState=OPEN, openSeqNum=2, regionLocation=91f82191efb8,41451,1731640420638 2024-11-15T03:13:44,567 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1731640423069.eaf85cc654192e46d3ea0904493eea12., pid=52, masterSystemTime=1731640424234 2024-11-15T03:13:44,569 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1731640423069.eaf85cc654192e46d3ea0904493eea12. 2024-11-15T03:13:44,569 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(153): Opened test,mmm,1731640423069.eaf85cc654192e46d3ea0904493eea12. 2024-11-15T03:13:44,569 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure e0f9e1e28ca8906f0f82f959a60c55ce, server=91f82191efb8,41451,1731640420638 because future has completed 2024-11-15T03:13:44,569 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(132): Open test,hhh,1731640423069.48ff514f5498660a4e8fc753340bb847. 2024-11-15T03:13:44,570 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7752): Opening region: {ENCODED => 48ff514f5498660a4e8fc753340bb847, NAME => 'test,hhh,1731640423069.48ff514f5498660a4e8fc753340bb847.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-15T03:13:44,570 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 48ff514f5498660a4e8fc753340bb847 2024-11-15T03:13:44,570 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(898): Instantiated test,hhh,1731640423069.48ff514f5498660a4e8fc753340bb847.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T03:13:44,570 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7794): checking encryption for 48ff514f5498660a4e8fc753340bb847 2024-11-15T03:13:44,570 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7797): checking classloading for 48ff514f5498660a4e8fc753340bb847 2024-11-15T03:13:44,572 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=10 2024-11-15T03:13:44,572 INFO [StoreOpener-48ff514f5498660a4e8fc753340bb847-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 48ff514f5498660a4e8fc753340bb847 2024-11-15T03:13:44,572 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure c4b611da4a73d4cfc99379713486e21c, server=91f82191efb8,37523,1731640420593 in 431 msec 2024-11-15T03:13:44,574 INFO [StoreOpener-48ff514f5498660a4e8fc753340bb847-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 48ff514f5498660a4e8fc753340bb847 columnFamilyName family 2024-11-15T03:13:44,574 DEBUG [StoreOpener-48ff514f5498660a4e8fc753340bb847-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T03:13:44,575 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=c4b611da4a73d4cfc99379713486e21c, ASSIGN in 761 msec 2024-11-15T03:13:44,575 INFO [StoreOpener-48ff514f5498660a4e8fc753340bb847-1 {}] regionserver.HStore(327): Store=48ff514f5498660a4e8fc753340bb847/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T03:13:44,575 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1038): replaying wal for 48ff514f5498660a4e8fc753340bb847 2024-11-15T03:13:44,576 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/48ff514f5498660a4e8fc753340bb847 2024-11-15T03:13:44,577 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/48ff514f5498660a4e8fc753340bb847 2024-11-15T03:13:44,578 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1048): stopping wal replay for 48ff514f5498660a4e8fc753340bb847 2024-11-15T03:13:44,578 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1060): Cleaning up temporary data for 48ff514f5498660a4e8fc753340bb847 2024-11-15T03:13:44,579 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=9 2024-11-15T03:13:44,579 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure e0f9e1e28ca8906f0f82f959a60c55ce, server=91f82191efb8,41451,1731640420638 in 484 msec 2024-11-15T03:13:44,580 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): writing seq id for 48ff514f5498660a4e8fc753340bb847 2024-11-15T03:13:44,582 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e0f9e1e28ca8906f0f82f959a60c55ce, ASSIGN in 768 msec 2024-11-15T03:13:44,583 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/48ff514f5498660a4e8fc753340bb847/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T03:13:44,583 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=eaf85cc654192e46d3ea0904493eea12, regionState=OPEN, openSeqNum=2, regionLocation=91f82191efb8,37523,1731640420593 2024-11-15T03:13:44,584 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1114): Opened 48ff514f5498660a4e8fc753340bb847; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63067679, jitterRate=-0.06021834909915924}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T03:13:44,585 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 48ff514f5498660a4e8fc753340bb847 2024-11-15T03:13:44,585 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1006): Region open journal for 48ff514f5498660a4e8fc753340bb847: Running coprocessor pre-open hook at 1731640424570Writing region info on filesystem at 1731640424570Initializing all the Stores at 1731640424572 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731640424572Cleaning up temporary data from old regions at 1731640424578 (+6 ms)Running coprocessor post-open hooks at 1731640424585 (+7 ms)Region opened successfully at 1731640424585 2024-11-15T03:13:44,586 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure eaf85cc654192e46d3ea0904493eea12, server=91f82191efb8,37523,1731640420593 because future has completed 2024-11-15T03:13:44,587 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1731640423069.48ff514f5498660a4e8fc753340bb847., pid=51, masterSystemTime=1731640424234 2024-11-15T03:13:44,589 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2266): Finished post open deploy task for test,hhh,1731640423069.48ff514f5498660a4e8fc753340bb847. 2024-11-15T03:13:44,590 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(153): Opened test,hhh,1731640423069.48ff514f5498660a4e8fc753340bb847. 2024-11-15T03:13:44,590 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=18 2024-11-15T03:13:44,590 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure a73d2d096e2f14a7e9365ac762927d55, server=91f82191efb8,42027,1731640420387 in 576 msec 2024-11-15T03:13:44,593 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN in 779 msec 2024-11-15T03:13:44,595 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=48ff514f5498660a4e8fc753340bb847, regionState=OPEN, openSeqNum=2, regionLocation=91f82191efb8,37523,1731640420593 2024-11-15T03:13:44,597 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 48ff514f5498660a4e8fc753340bb847, server=91f82191efb8,37523,1731640420593 because future has completed 2024-11-15T03:13:44,601 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=17 2024-11-15T03:13:44,601 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure eaf85cc654192e46d3ea0904493eea12, server=91f82191efb8,37523,1731640420593 in 476 msec 2024-11-15T03:13:44,603 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=eaf85cc654192e46d3ea0904493eea12, ASSIGN in 790 msec 2024-11-15T03:13:44,607 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=12 2024-11-15T03:13:44,607 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure 48ff514f5498660a4e8fc753340bb847, server=91f82191efb8,37523,1731640420593 in 485 msec 2024-11-15T03:13:44,609 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=48ff514f5498660a4e8fc753340bb847, ASSIGN in 796 msec 2024-11-15T03:13:44,613 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=4 2024-11-15T03:13:44,613 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN in 798 msec 2024-11-15T03:13:44,614 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T03:13:44,615 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731640424615"}]},"ts":"1731640424615"} 2024-11-15T03:13:44,618 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLED in hbase:meta 2024-11-15T03:13:44,620 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T03:13:44,627 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test in 1.5400 sec 2024-11-15T03:13:45,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42051 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T03:13:45,244 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test completed 2024-11-15T03:13:45,244 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table test get assigned. Timeout = 60000ms 2024-11-15T03:13:45,247 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-15T03:13:45,271 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table test assigned to meta. Checking AM states. 2024-11-15T03:13:45,271 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-15T03:13:45,272 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table test assigned. 2024-11-15T03:13:45,272 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-15T03:13:45,278 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-15T03:13:45,308 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-15T03:13:45,324 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T03:13:45,326 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38882, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T03:13:45,329 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T03:13:45,331 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57744, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T03:13:45,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42051 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 6130297c45bd23340b368de2cc708c90 changed from -1.0 to 0.0, refreshing cache 2024-11-15T03:13:45,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42051 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c4b611da4a73d4cfc99379713486e21c changed from -1.0 to 0.0, refreshing cache 2024-11-15T03:13:45,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42051 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 48ff514f5498660a4e8fc753340bb847 changed from -1.0 to 0.0, refreshing cache 2024-11-15T03:13:45,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42051 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region b1704275babe02e36b29cfaadf5b81af changed from -1.0 to 0.0, refreshing cache 2024-11-15T03:13:45,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42051 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region eaf85cc654192e46d3ea0904493eea12 changed from -1.0 to 0.0, refreshing cache 2024-11-15T03:13:45,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42051 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region fac2a1b464a5ff2d581af6baef327bde changed from -1.0 to 0.0, refreshing cache 2024-11-15T03:13:45,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42051 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 21df709a703798d68290a0a63322957f changed from -1.0 to 0.0, refreshing cache 2024-11-15T03:13:45,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42051 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 155ef591549501ad2da24d2c559fb7f9 changed from -1.0 to 0.0, refreshing cache 2024-11-15T03:13:45,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42051 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 17e8b3b889a606b8c6826552e784d401 changed from -1.0 to 0.0, refreshing cache 2024-11-15T03:13:45,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42051 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 5965b596ce7b87fabde47569d60c3f95 changed from -1.0 to 0.0, refreshing cache 2024-11-15T03:13:45,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42051 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region e0f9e1e28ca8906f0f82f959a60c55ce changed from -1.0 to 0.0, refreshing cache 2024-11-15T03:13:45,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42051 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f5219cfe492c74eb8312930d328964b6 changed from -1.0 to 0.0, refreshing cache 2024-11-15T03:13:45,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42051 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 860df8edb65f2fdd3c8bc00f89f94ba0 changed from -1.0 to 0.0, refreshing cache 2024-11-15T03:13:45,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42051 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region b96380a8d7110dffa8cd7844252025dd changed from -1.0 to 0.0, refreshing cache 2024-11-15T03:13:45,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42051 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region dc930ecfca79abbf57ed28c20f79230f changed from -1.0 to 0.0, refreshing cache 2024-11-15T03:13:45,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42051 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 4ef03d91ad777a77cf66f510e28419ba changed from -1.0 to 0.0, refreshing cache 2024-11-15T03:13:45,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42051 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 6cca2d3f6a001d96f34b7a93afec8fe7 changed from -1.0 to 0.0, refreshing cache 2024-11-15T03:13:45,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42051 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 5971f307349447e7f24b0dd05b96cffc changed from -1.0 to 0.0, refreshing cache 2024-11-15T03:13:45,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42051 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1beb579fcf8e161a0b9c2051c671a8f2 changed from -1.0 to 0.0, refreshing cache 2024-11-15T03:13:45,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42051 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region d0eff1b91bc9b77579a5c44f1c985985 changed from -1.0 to 0.0, refreshing cache 2024-11-15T03:13:45,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42051 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region bb2ead3b463edc0f2b603da7d19f1201 changed from -1.0 to 0.0, refreshing cache 2024-11-15T03:13:45,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42051 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region a73d2d096e2f14a7e9365ac762927d55 changed from -1.0 to 0.0, refreshing cache 2024-11-15T03:13:45,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42051 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 23f804d67a47ba185ffd6f6bc7b75464 changed from -1.0 to 0.0, refreshing cache 2024-11-15T03:13:45,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42051 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1b8f1ea215e8c2f78186b80dafed1b28 changed from -1.0 to 0.0, refreshing cache 2024-11-15T03:13:45,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42051 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region dc93af816bdf94cf920a5adc31356783 changed from -1.0 to 0.0, refreshing cache 2024-11-15T03:13:45,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42051 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region e0bce1a97a9c7f320b315db443b47c23 changed from -1.0 to 0.0, refreshing cache 2024-11-15T03:13:45,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42051 {}] rsgroup.RSGroupBasedLoadBalancer(138): Balancing RSGroup=default 2024-11-15T03:13:45,400 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42051 {}] rsgroup.RSGroupBasedLoadBalancer(151): Start Generate Balance plan for group: default 2024-11-15T03:13:45,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42051 {}] balancer.BaseLoadBalancer(619): Start Generate Balance plan for cluster. 2024-11-15T03:13:45,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42051 {}] balancer.BalancerClusterState(204): Hosts are {91f82191efb8=0} racks are {/default-rack=0} 2024-11-15T03:13:45,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42051 {}] balancer.BalancerClusterState(303): server 0 has 9 regions 2024-11-15T03:13:45,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42051 {}] balancer.BalancerClusterState(303): server 1 has 8 regions 2024-11-15T03:13:45,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42051 {}] balancer.BalancerClusterState(303): server 2 has 10 regions 2024-11-15T03:13:45,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42051 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-15T03:13:45,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42051 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-15T03:13:45,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42051 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-15T03:13:45,401 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42051 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-15T03:13:45,401 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42051 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-15T03:13:45,402 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42051 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-15T03:13:45,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42051 {}] balancer.BalancerClusterState(326): Number of tables=2, number of hosts=1, number of racks=1 2024-11-15T03:13:45,405 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42051 {}] balancer.StochasticLoadBalancer(403): Cluster wide - Calculating plan. may take up to 30000ms to complete. 2024-11-15T03:13:45,406 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42051 {}] balancer.StochasticLoadBalancer(515): Start StochasticLoadBalancer.balancer, initial weighted average imbalance=0.06931633435746164, functionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.06415002990995843, need balance); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.0); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); computedMaxSteps=64800 2024-11-15T03:13:45,918 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42051 {}] balancer.StochasticLoadBalancer(562): Finished computing new moving plan. Computation took 515 ms to try 64800 different iterations. Found a solution that moves 1 regions; Going from a computed imbalance of 0.06931633435746164 to a new imbalance of 0.01690158032826896. funtionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.0); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.037037037037037035, need balance); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); 2024-11-15T03:13:45,921 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42051 {}] master.HMaster(2167): Balancer plans size is 1, the balance interval is 300000 ms, and the max number regions in transition is 27 2024-11-15T03:13:45,921 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42051 {}] master.HMaster(2172): balance hri=dc93af816bdf94cf920a5adc31356783, source=91f82191efb8,42027,1731640420387, destination=91f82191efb8,41451,1731640420638 2024-11-15T03:13:45,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42051 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test, region=dc93af816bdf94cf920a5adc31356783, REOPEN/MOVE 2024-11-15T03:13:45,923 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test, region=dc93af816bdf94cf920a5adc31356783, REOPEN/MOVE 2024-11-15T03:13:45,925 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=dc93af816bdf94cf920a5adc31356783, regionState=CLOSING, regionLocation=91f82191efb8,42027,1731640420387 2024-11-15T03:13:45,927 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=test, region=dc93af816bdf94cf920a5adc31356783, REOPEN/MOVE because future has completed 2024-11-15T03:13:45,928 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T03:13:45,928 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE, hasLock=false; CloseRegionProcedure dc93af816bdf94cf920a5adc31356783, server=91f82191efb8,42027,1731640420387}] 2024-11-15T03:13:46,088 INFO [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(122): Close dc93af816bdf94cf920a5adc31356783 2024-11-15T03:13:46,088 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T03:13:46,090 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1722): Closing dc93af816bdf94cf920a5adc31356783, disabling compactions & flushes 2024-11-15T03:13:46,090 INFO [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1755): Closing region test,vvv,1731640423069.dc93af816bdf94cf920a5adc31356783. 2024-11-15T03:13:46,090 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1731640423069.dc93af816bdf94cf920a5adc31356783. 2024-11-15T03:13:46,090 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1731640423069.dc93af816bdf94cf920a5adc31356783. after waiting 0 ms 2024-11-15T03:13:46,090 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1731640423069.dc93af816bdf94cf920a5adc31356783. 2024-11-15T03:13:46,097 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/dc93af816bdf94cf920a5adc31356783/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T03:13:46,100 INFO [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1973): Closed test,vvv,1731640423069.dc93af816bdf94cf920a5adc31356783. 2024-11-15T03:13:46,100 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1676): Region close journal for dc93af816bdf94cf920a5adc31356783: Waiting for close lock at 1731640426089Running coprocessor pre-close hooks at 1731640426089Disabling compacts and flushes for region at 1731640426089Disabling writes for close at 1731640426090 (+1 ms)Writing region close event to WAL at 1731640426091 (+1 ms)Running coprocessor post-close hooks at 1731640426098 (+7 ms)Closed at 1731640426100 (+2 ms) 2024-11-15T03:13:46,100 INFO [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegionServer(3302): Adding dc93af816bdf94cf920a5adc31356783 move to 91f82191efb8,41451,1731640420638 record at close sequenceid=2 2024-11-15T03:13:46,105 INFO [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(157): Closed dc93af816bdf94cf920a5adc31356783 2024-11-15T03:13:46,105 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=dc93af816bdf94cf920a5adc31356783, regionState=CLOSED 2024-11-15T03:13:46,108 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE, hasLock=false; CloseRegionProcedure dc93af816bdf94cf920a5adc31356783, server=91f82191efb8,42027,1731640420387 because future has completed 2024-11-15T03:13:46,112 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-15T03:13:46,112 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; CloseRegionProcedure dc93af816bdf94cf920a5adc31356783, server=91f82191efb8,42027,1731640420387 in 181 msec 2024-11-15T03:13:46,114 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=dc93af816bdf94cf920a5adc31356783, REOPEN/MOVE; state=CLOSED, location=91f82191efb8,41451,1731640420638; forceNewPlan=false, retain=false 2024-11-15T03:13:46,264 INFO [91f82191efb8:42051 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-15T03:13:46,265 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=dc93af816bdf94cf920a5adc31356783, regionState=OPENING, regionLocation=91f82191efb8,41451,1731640420638 2024-11-15T03:13:46,269 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=dc93af816bdf94cf920a5adc31356783, REOPEN/MOVE because future has completed 2024-11-15T03:13:46,270 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=57, state=RUNNABLE, hasLock=false; OpenRegionProcedure dc93af816bdf94cf920a5adc31356783, server=91f82191efb8,41451,1731640420638}] 2024-11-15T03:13:46,430 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(132): Open test,vvv,1731640423069.dc93af816bdf94cf920a5adc31356783. 2024-11-15T03:13:46,431 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7752): Opening region: {ENCODED => dc93af816bdf94cf920a5adc31356783, NAME => 'test,vvv,1731640423069.dc93af816bdf94cf920a5adc31356783.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-15T03:13:46,431 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test dc93af816bdf94cf920a5adc31356783 2024-11-15T03:13:46,431 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(898): Instantiated test,vvv,1731640423069.dc93af816bdf94cf920a5adc31356783.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T03:13:46,432 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7794): checking encryption for dc93af816bdf94cf920a5adc31356783 2024-11-15T03:13:46,432 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7797): checking classloading for dc93af816bdf94cf920a5adc31356783 2024-11-15T03:13:46,435 INFO [StoreOpener-dc93af816bdf94cf920a5adc31356783-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region dc93af816bdf94cf920a5adc31356783 2024-11-15T03:13:46,436 INFO [StoreOpener-dc93af816bdf94cf920a5adc31356783-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dc93af816bdf94cf920a5adc31356783 columnFamilyName family 2024-11-15T03:13:46,437 DEBUG [StoreOpener-dc93af816bdf94cf920a5adc31356783-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T03:13:46,438 INFO [StoreOpener-dc93af816bdf94cf920a5adc31356783-1 {}] regionserver.HStore(327): Store=dc93af816bdf94cf920a5adc31356783/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T03:13:46,438 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1038): replaying wal for dc93af816bdf94cf920a5adc31356783 2024-11-15T03:13:46,440 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/dc93af816bdf94cf920a5adc31356783 2024-11-15T03:13:46,441 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/dc93af816bdf94cf920a5adc31356783 2024-11-15T03:13:46,442 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1048): stopping wal replay for dc93af816bdf94cf920a5adc31356783 2024-11-15T03:13:46,442 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1060): Cleaning up temporary data for dc93af816bdf94cf920a5adc31356783 2024-11-15T03:13:46,444 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1093): writing seq id for dc93af816bdf94cf920a5adc31356783 2024-11-15T03:13:46,445 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1114): Opened dc93af816bdf94cf920a5adc31356783; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61111685, jitterRate=-0.0893649309873581}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T03:13:46,446 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1122): Running coprocessor post-open hooks for dc93af816bdf94cf920a5adc31356783 2024-11-15T03:13:46,446 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1006): Region open journal for dc93af816bdf94cf920a5adc31356783: Running coprocessor pre-open hook at 1731640426432Writing region info on filesystem at 1731640426432Initializing all the Stores at 1731640426434 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731640426434Cleaning up temporary data from old regions at 1731640426442 (+8 ms)Running coprocessor post-open hooks at 1731640426446 (+4 ms)Region opened successfully at 1731640426446 2024-11-15T03:13:46,447 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1731640423069.dc93af816bdf94cf920a5adc31356783., pid=59, masterSystemTime=1731640426424 2024-11-15T03:13:46,450 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1731640423069.dc93af816bdf94cf920a5adc31356783. 2024-11-15T03:13:46,450 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(153): Opened test,vvv,1731640423069.dc93af816bdf94cf920a5adc31356783. 2024-11-15T03:13:46,450 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=dc93af816bdf94cf920a5adc31356783, regionState=OPEN, openSeqNum=5, regionLocation=91f82191efb8,41451,1731640420638 2024-11-15T03:13:46,453 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=57, state=RUNNABLE, hasLock=false; OpenRegionProcedure dc93af816bdf94cf920a5adc31356783, server=91f82191efb8,41451,1731640420638 because future has completed 2024-11-15T03:13:46,458 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=57 2024-11-15T03:13:46,458 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=57, state=SUCCESS, hasLock=false; OpenRegionProcedure dc93af816bdf94cf920a5adc31356783, server=91f82191efb8,41451,1731640420638 in 185 msec 2024-11-15T03:13:46,460 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=dc93af816bdf94cf920a5adc31356783, REOPEN/MOVE in 537 msec 2024-11-15T03:13:46,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42051 {}] master.HMaster(2203): Balancer is going into sleep until next period in 300000ms 2024-11-15T03:13:46,526 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [900,000] milli-secs(wait.for.ratio=[1]) 2024-11-15T03:13:46,533 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42051 {}] master.MasterRpcServices(567): Client=jenkins//172.17.0.2 set balanceSwitch=false 2024-11-15T03:13:46,547 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.assignment.TestRollbackSCP#testFailAndRollback Thread=384, OpenFileDescriptor=650, MaxFileDescriptor=1048576, SystemLoadAverage=277, ProcessCount=11, AvailableMemoryMB=2998 2024-11-15T03:13:46,550 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table hbase:meta 2024-11-15T03:13:46,551 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=hbase:meta,,1 2024-11-15T03:13:46,551 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing 91f82191efb8,42027,1731640420387 2024-11-15T03:13:46,552 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 91f82191efb8,42027,1731640420387: Simulated kill ***** 2024-11-15T03:13:46,552 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-15T03:13:46,554 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-15T03:13:46,557 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-15T03:13:46,558 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-15T03:13:46,558 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-15T03:13:46,558 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 335289152 }, "NonHeapMemoryUsage": { "committed": 153812992, "init": 7667712, "max": -1, "used": 148440696 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "91f82191efb8" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "91f82191efb8" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.Context": "regionserver", "tag.Hostname": "91f82191efb8" } ] } 2024-11-15T03:13:46,561 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42051 {}] master.MasterRpcServices(700): 91f82191efb8,42027,1731640420387 reported a fatal error: ***** ABORTING region server 91f82191efb8,42027,1731640420387: Simulated kill ***** 2024-11-15T03:13:46,563 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '91f82191efb8,42027,1731640420387' ***** 2024-11-15T03:13:46,563 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-11-15T03:13:46,563 INFO [RS:0;91f82191efb8:42027 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-15T03:13:46,563 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [15,000] milli-secs(wait.for.ratio=[1]) 2024-11-15T03:13:46,564 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-15T03:13:46,564 INFO [RS:0;91f82191efb8:42027 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-15T03:13:46,564 INFO [RS:0;91f82191efb8:42027 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-15T03:13:46,564 INFO [RS:0;91f82191efb8:42027 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-15T03:13:46,564 INFO [RS:0;91f82191efb8:42027 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;91f82191efb8:42027. 2024-11-15T03:13:46,564 DEBUG [RS:0;91f82191efb8:42027 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T03:13:46,569 DEBUG [RS:0;91f82191efb8:42027 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T03:13:46,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36559 is added to blk_1073741836_1012 (size=33749) 2024-11-15T03:13:46,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46785 is added to blk_1073741836_1012 (size=33749) 2024-11-15T03:13:46,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38319 is added to blk_1073741836_1012 (size=33749) 2024-11-15T03:13:46,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46785 is added to blk_1073741835_1011 (size=2774) 2024-11-15T03:13:46,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36559 is added to blk_1073741835_1011 (size=2774) 2024-11-15T03:13:46,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38319 is added to blk_1073741835_1011 (size=2774) 2024-11-15T03:13:46,580 DEBUG [RS:0;91f82191efb8:42027 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T03:13:46,580 INFO [RS:0;91f82191efb8:42027 {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T03:13:46,581 INFO [RS:0;91f82191efb8:42027 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42027 2024-11-15T03:13:46,648 INFO [regionserver/91f82191efb8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T03:13:46,651 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42027-0x1013c7bdd830001, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/91f82191efb8,42027,1731640420387 2024-11-15T03:13:46,651 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42051-0x1013c7bdd830000, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-15T03:13:46,651 INFO [RS:0;91f82191efb8:42027 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-15T03:13:46,662 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [91f82191efb8,42027,1731640420387] 2024-11-15T03:13:46,672 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/91f82191efb8,42027,1731640420387 already deleted, retry=false 2024-11-15T03:13:46,672 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 91f82191efb8,42027,1731640420387 on 91f82191efb8,42051,1731640419585 2024-11-15T03:13:46,678 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true 2024-11-15T03:13:46,682 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=60, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true 2024-11-15T03:13:46,682 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1991): Scheduled ServerCrashProcedure pid=60 for 91f82191efb8,42027,1731640420387 (carryingMeta=true) 91f82191efb8,42027,1731640420387/CRASHED/regionCount=9/lock=java.util.concurrent.locks.ReentrantReadWriteLock@2ec75011[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-15T03:13:46,683 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=60, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true, isMeta: true 2024-11-15T03:13:46,686 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/WALs/91f82191efb8,42027,1731640420387-splitting 2024-11-15T03:13:46,687 INFO [PEWorker-1 {}] master.SplitWALManager(105): 91f82191efb8,42027,1731640420387 WAL count=1, meta=true 2024-11-15T03:13:46,690 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta}] 2024-11-15T03:13:46,695 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:46,697 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593}] 2024-11-15T03:13:46,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42027-0x1013c7bdd830001, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T03:13:46,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42027-0x1013c7bdd830001, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T03:13:46,762 INFO [RS:0;91f82191efb8:42027 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-15T03:13:46,763 INFO [RS:0;91f82191efb8:42027 {}] regionserver.HRegionServer(1031): Exiting; stopping=91f82191efb8,42027,1731640420387; zookeeper connection closed. 2024-11-15T03:13:46,763 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@21df879f {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@21df879f 2024-11-15T03:13:46,765 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-15T03:13:46,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37523 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=62 2024-11-15T03:13:46,886 INFO [RS_LOG_REPLAY_OPS-regionserver/91f82191efb8:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(299): Splitting hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/WALs/91f82191efb8,42027,1731640420387-splitting/91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, size=33.0 K (33749bytes) 2024-11-15T03:13:46,887 DEBUG [RS_LOG_REPLAY_OPS-regionserver/91f82191efb8:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-15T03:13:46,887 DEBUG [RS_LOG_REPLAY_OPS-regionserver/91f82191efb8:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-15T03:13:46,888 INFO [RS_LOG_REPLAY_OPS-regionserver/91f82191efb8:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/WALs/91f82191efb8,42027,1731640420387-splitting/91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta 2024-11-15T03:13:46,893 INFO [RS_LOG_REPLAY_OPS-regionserver/91f82191efb8:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/WALs/91f82191efb8,42027,1731640420387-splitting/91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta after 3ms 2024-11-15T03:13:46,901 INFO [RS_LOG_REPLAY_OPS-regionserver/91f82191efb8:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(310): Open hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/WALs/91f82191efb8,42027,1731640420387-splitting/91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta took 16ms 2024-11-15T03:13:46,910 DEBUG [RS_LOG_REPLAY_OPS-regionserver/91f82191efb8:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-15T03:13:46,916 INFO [RS_LOG_REPLAY_OPS-regionserver/91f82191efb8:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta.temp 2024-11-15T03:13:46,919 INFO [RS_LOG_REPLAY_OPS-regionserver/91f82191efb8:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta.temp 2024-11-15T03:13:46,923 DEBUG [RS_LOG_REPLAY_OPS-regionserver/91f82191efb8:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/WALs/91f82191efb8,42027,1731640420387-splitting/91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta so closing down 2024-11-15T03:13:46,923 DEBUG [RS_LOG_REPLAY_OPS-regionserver/91f82191efb8:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-15T03:13:46,924 INFO [RS_LOG_REPLAY_OPS-regionserver/91f82191efb8:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.OutputSink(145): 3 split writer threads finished 2024-11-15T03:13:46,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46785 is added to blk_1073741864_1040 (size=33508) 2024-11-15T03:13:46,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36559 is added to blk_1073741864_1040 (size=33508) 2024-11-15T03:13:46,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38319 is added to blk_1073741864_1040 (size=33508) 2024-11-15T03:13:46,936 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta.temp (wrote 61 edits, skipped 0 edits in 4 ms) 2024-11-15T03:13:46,938 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta.temp to hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-15T03:13:46,942 INFO [RS_LOG_REPLAY_OPS-regionserver/91f82191efb8:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(425): Processed 62 edits across 1 Regions in 36 ms; skipped=1; WAL=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/WALs/91f82191efb8,42027,1731640420387-splitting/91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, size=33.0 K, length=33749, corrupted=false, cancelled=false 2024-11-15T03:13:46,942 DEBUG [RS_LOG_REPLAY_OPS-regionserver/91f82191efb8:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(428): Completed split of hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/WALs/91f82191efb8,42027,1731640420387-splitting/91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, journal: Splitting hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/WALs/91f82191efb8,42027,1731640420387-splitting/91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, size=33.0 K (33749bytes) at 1731640426886Creating recovered edits writer path=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta.temp at 1731640426919 (+33 ms)Finishing writing output for hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/WALs/91f82191efb8,42027,1731640420387-splitting/91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta so closing down at 1731640426923 (+4 ms)3 split writer threads finished at 1731640426924 (+1 ms)Closed recovered edits writer path=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta.temp (wrote 61 edits, skipped 0 edits in 4 ms) at 1731640426937 (+13 ms)Rename recovered edits hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta.temp to hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 at 1731640426938 (+1 ms)Processed 62 edits across 1 Regions in 36 ms; skipped=1; WAL=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/WALs/91f82191efb8,42027,1731640420387-splitting/91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, size=33.0 K, length=33749, corrupted=false, cancelled=false at 1731640426942 (+4 ms) 2024-11-15T03:13:46,942 DEBUG [RS_LOG_REPLAY_OPS-regionserver/91f82191efb8:0-0 {event_type=RS_LOG_REPLAY, pid=62}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/WALs/91f82191efb8,42027,1731640420387-splitting/91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta 2024-11-15T03:13:46,943 DEBUG [RS_LOG_REPLAY_OPS-regionserver/91f82191efb8:0-0 {event_type=RS_LOG_REPLAY, pid=62}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=62 2024-11-15T03:13:46,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42051 {}] master.HMaster(4169): Remote procedure done, pid=62 2024-11-15T03:13:46,952 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/WALs/91f82191efb8,42027,1731640420387-splitting/91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta to hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/oldWALs 2024-11-15T03:13:46,955 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-15T03:13:46,955 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 in 255 msec 2024-11-15T03:13:46,957 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:46,959 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-15T03:13:46,959 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 in 268 msec 2024-11-15T03:13:46,961 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/WALs/91f82191efb8,42027,1731640420387-splitting dir is empty, no logs to split. 2024-11-15T03:13:46,961 INFO [PEWorker-1 {}] master.SplitWALManager(105): 91f82191efb8,42027,1731640420387 WAL count=0, meta=true 2024-11-15T03:13:46,962 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 91f82191efb8,42027,1731640420387 WAL splitting is done? wals=0, meta=true 2024-11-15T03:13:46,963 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-15T03:13:46,965 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T03:13:46,966 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-15T03:13:47,114 INFO [regionserver/91f82191efb8:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-15T03:13:47,114 INFO [regionserver/91f82191efb8:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-15T03:13:47,116 DEBUG [91f82191efb8:42051 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-15T03:13:47,116 DEBUG [91f82191efb8:42051 {}] balancer.BalancerClusterState(204): Hosts are {91f82191efb8=0} racks are {/default-rack=0} 2024-11-15T03:13:47,117 DEBUG [91f82191efb8:42051 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-15T03:13:47,117 DEBUG [91f82191efb8:42051 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-15T03:13:47,117 DEBUG [91f82191efb8:42051 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-15T03:13:47,117 DEBUG [91f82191efb8:42051 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-15T03:13:47,117 INFO [91f82191efb8:42051 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-15T03:13:47,117 INFO [91f82191efb8:42051 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-15T03:13:47,117 DEBUG [91f82191efb8:42051 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-15T03:13:47,118 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=63 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=91f82191efb8,37523,1731640420593 2024-11-15T03:13:47,119 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 91f82191efb8,37523,1731640420593, state=OPENING 2024-11-15T03:13:47,141 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37523-0x1013c7bdd830002, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:47,141 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41451-0x1013c7bdd830003, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:47,141 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42051-0x1013c7bdd830000, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:47,141 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T03:13:47,141 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:47,141 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:47,141 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:47,142 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=91f82191efb8,37523,1731640420593}] 2024-11-15T03:13:47,308 INFO [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=64}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-15T03:13:47,308 INFO [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T03:13:47,309 INFO [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=64}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-15T03:13:47,311 INFO [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=91f82191efb8%2C37523%2C1731640420593.meta, suffix=.meta, logDir=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/WALs/91f82191efb8,37523,1731640420593, archiveDir=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/oldWALs, maxLogs=32 2024-11-15T03:13:47,326 DEBUG [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=64}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/WALs/91f82191efb8,37523,1731640420593/91f82191efb8%2C37523%2C1731640420593.meta.1731640427312.meta, exclude list is [], retry=0 2024-11-15T03:13:47,329 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38319,DS-e648626c-dc65-44f2-ab4f-9b6acd52971a,DISK] 2024-11-15T03:13:47,329 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46785,DS-0b930937-9ceb-4d09-87e5-60bb062fe96b,DISK] 2024-11-15T03:13:47,329 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36559,DS-818fe711-350c-400c-9163-61e8ae4f0904,DISK] 2024-11-15T03:13:47,332 INFO [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/WALs/91f82191efb8,37523,1731640420593/91f82191efb8%2C37523%2C1731640420593.meta.1731640427312.meta 2024-11-15T03:13:47,338 DEBUG [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36035:36035),(127.0.0.1/127.0.0.1:37901:37901),(127.0.0.1/127.0.0.1:45153:45153)] 2024-11-15T03:13:47,338 DEBUG [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-15T03:13:47,338 DEBUG [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=64}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-15T03:13:47,338 DEBUG [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-15T03:13:47,338 INFO [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-15T03:13:47,338 DEBUG [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-15T03:13:47,339 DEBUG [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T03:13:47,339 DEBUG [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-15T03:13:47,339 DEBUG [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-15T03:13:47,340 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-15T03:13:47,343 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-15T03:13:47,343 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T03:13:47,343 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T03:13:47,343 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-15T03:13:47,344 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-15T03:13:47,345 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T03:13:47,345 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T03:13:47,345 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-15T03:13:47,346 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-15T03:13:47,346 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T03:13:47,347 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T03:13:47,347 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-15T03:13:47,348 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-15T03:13:47,348 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T03:13:47,348 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T03:13:47,348 DEBUG [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-15T03:13:47,349 DEBUG [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/hbase/meta/1588230740 2024-11-15T03:13:47,352 DEBUG [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/hbase/meta/1588230740 2024-11-15T03:13:47,353 INFO [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-15T03:13:47,370 DEBUG [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5793): Applied 277, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=64, path=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-15T03:13:47,373 INFO [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=29.41 KB heapSize=50.67 KB 2024-11-15T03:13:47,442 DEBUG [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/hbase/meta/1588230740/.tmp/info/61817a12d7984da5b83f8a5be284fbc2 is 138, key is test,bbb,1731640423069.6130297c45bd23340b368de2cc708c90./info:regioninfo/1731640424530/Put/seqid=0 2024-11-15T03:13:47,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36559 is added to blk_1073741866_1042 (size=36381) 2024-11-15T03:13:47,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46785 is added to blk_1073741866_1042 (size=36381) 2024-11-15T03:13:47,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38319 is added to blk_1073741866_1042 (size=36381) 2024-11-15T03:13:47,464 INFO [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.26 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/hbase/meta/1588230740/.tmp/info/61817a12d7984da5b83f8a5be284fbc2 2024-11-15T03:13:47,524 DEBUG [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/hbase/meta/1588230740/.tmp/ns/d0453fed43324a8fa6742b928c34132f is 43, key is default/ns:d/1731640422871/Put/seqid=0 2024-11-15T03:13:47,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46785 is added to blk_1073741867_1043 (size=5153) 2024-11-15T03:13:47,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38319 is added to blk_1073741867_1043 (size=5153) 2024-11-15T03:13:47,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36559 is added to blk_1073741867_1043 (size=5153) 2024-11-15T03:13:47,535 INFO [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/hbase/meta/1588230740/.tmp/ns/d0453fed43324a8fa6742b928c34132f 2024-11-15T03:13:47,560 DEBUG [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/hbase/meta/1588230740/.tmp/table/6dfa0cdbbee84063a72b2f5b31e784da is 40, key is test/table:state/1731640424615/Put/seqid=0 2024-11-15T03:13:47,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46785 is added to blk_1073741868_1044 (size=5165) 2024-11-15T03:13:47,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36559 is added to blk_1073741868_1044 (size=5165) 2024-11-15T03:13:47,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38319 is added to blk_1073741868_1044 (size=5165) 2024-11-15T03:13:47,571 INFO [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=72 B at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/hbase/meta/1588230740/.tmp/table/6dfa0cdbbee84063a72b2f5b31e784da 2024-11-15T03:13:47,582 DEBUG [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/hbase/meta/1588230740/.tmp/info/61817a12d7984da5b83f8a5be284fbc2 as hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/hbase/meta/1588230740/info/61817a12d7984da5b83f8a5be284fbc2 2024-11-15T03:13:47,592 INFO [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/hbase/meta/1588230740/info/61817a12d7984da5b83f8a5be284fbc2, entries=273, sequenceid=64, filesize=35.5 K 2024-11-15T03:13:47,595 DEBUG [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/hbase/meta/1588230740/.tmp/ns/d0453fed43324a8fa6742b928c34132f as hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/hbase/meta/1588230740/ns/d0453fed43324a8fa6742b928c34132f 2024-11-15T03:13:47,604 INFO [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/hbase/meta/1588230740/ns/d0453fed43324a8fa6742b928c34132f, entries=2, sequenceid=64, filesize=5.0 K 2024-11-15T03:13:47,606 DEBUG [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/hbase/meta/1588230740/.tmp/table/6dfa0cdbbee84063a72b2f5b31e784da as hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/hbase/meta/1588230740/table/6dfa0cdbbee84063a72b2f5b31e784da 2024-11-15T03:13:47,615 INFO [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/hbase/meta/1588230740/table/6dfa0cdbbee84063a72b2f5b31e784da, entries=2, sequenceid=64, filesize=5.0 K 2024-11-15T03:13:47,621 INFO [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(3140): Finished flush of dataSize ~29.41 KB/30111, heapSize ~50.38 KB/51584, currentSize=0 B/0 for 1588230740 in 245ms, sequenceid=64, compaction requested=false; wal=null 2024-11-15T03:13:47,622 DEBUG [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-15T03:13:47,626 DEBUG [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-15T03:13:47,628 DEBUG [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-15T03:13:47,628 DEBUG [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-15T03:13:47,628 DEBUG [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-15T03:13:47,630 DEBUG [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-15T03:13:47,633 DEBUG [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/hbase/meta/1588230740/recovered.edits/64.seqid, newMaxSeqId=64, maxSeqId=1 2024-11-15T03:13:47,635 INFO [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=65; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61833035, jitterRate=-0.07861597836017609}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-15T03:13:47,635 DEBUG [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-15T03:13:47,635 DEBUG [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731640427339Writing region info on filesystem at 1731640427339Initializing all the Stores at 1731640427340 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731640427340Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731640427340Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731640427340Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731640427340Obtaining lock to block concurrent updates at 1731640427373 (+33 ms)Preparing flush snapshotting stores in 1588230740 at 1731640427373Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=30111, getHeapSize=51824, getOffHeapSize=0, getCellsCount=277 at 1731640427376 (+3 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731640427376Flushing 1588230740/info: creating writer at 1731640427377 (+1 ms)Flushing 1588230740/info: appending metadata at 1731640427430 (+53 ms)Flushing 1588230740/info: closing flushed file at 1731640427433 (+3 ms)Flushing 1588230740/ns: creating writer at 1731640427508 (+75 ms)Flushing 1588230740/ns: appending metadata at 1731640427523 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1731640427523Flushing 1588230740/table: creating writer at 1731640427544 (+21 ms)Flushing 1588230740/table: appending metadata at 1731640427560 (+16 ms)Flushing 1588230740/table: closing flushed file at 1731640427560Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2931c18c: reopening flushed file at 1731640427580 (+20 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@389fb14f: reopening flushed file at 1731640427594 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6e1f211f: reopening flushed file at 1731640427604 (+10 ms)Finished flush of dataSize ~29.41 KB/30111, heapSize ~50.38 KB/51584, currentSize=0 B/0 for 1588230740 in 245ms, sequenceid=64, compaction requested=false; wal=null at 1731640427621 (+17 ms)Cleaning up temporary data from old regions at 1731640427628 (+7 ms)Running coprocessor post-open hooks at 1731640427635 (+7 ms)Region opened successfully at 1731640427635 2024-11-15T03:13:47,636 INFO [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=64, masterSystemTime=1731640427296 2024-11-15T03:13:47,639 DEBUG [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-15T03:13:47,639 INFO [RS_OPEN_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_META, pid=64}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-15T03:13:47,640 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=63 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=65, regionLocation=91f82191efb8,37523,1731640420593 2024-11-15T03:13:47,641 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 91f82191efb8,37523,1731640420593, state=OPEN 2024-11-15T03:13:47,667 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42051 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=91f82191efb8,37523,1731640420593, table=hbase:meta, region=1588230740. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T03:13:47,693 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37523-0x1013c7bdd830002, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:47,693 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41451-0x1013c7bdd830003, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:47,693 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42051-0x1013c7bdd830000, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:47,693 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=64, ppid=63, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=91f82191efb8,37523,1731640420593 2024-11-15T03:13:47,693 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:47,693 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:47,693 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:47,696 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-11-15T03:13:47,696 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=91f82191efb8,37523,1731640420593 in 552 msec 2024-11-15T03:13:47,699 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=60 2024-11-15T03:13:47,699 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 733 msec 2024-11-15T03:13:47,699 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 91f82191efb8,42027,1731640420387 had 9 regions 2024-11-15T03:13:47,701 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=60, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true, isMeta: false 2024-11-15T03:13:47,702 INFO [PEWorker-1 {}] master.SplitWALManager(105): 91f82191efb8,42027,1731640420387 WAL count=1, meta=false 2024-11-15T03:13:47,703 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=60, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194}] 2024-11-15T03:13:47,705 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:47,706 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593}] 2024-11-15T03:13:47,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37523 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=66 2024-11-15T03:13:47,877 INFO [RS_LOG_REPLAY_OPS-regionserver/91f82191efb8:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(299): Splitting hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/WALs/91f82191efb8,42027,1731640420387-splitting/91f82191efb8%2C42027%2C1731640420387.1731640422194, size=2.7 K (2774bytes) 2024-11-15T03:13:47,877 INFO [RS_LOG_REPLAY_OPS-regionserver/91f82191efb8:0-1 {event_type=RS_LOG_REPLAY, pid=66}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/WALs/91f82191efb8,42027,1731640420387-splitting/91f82191efb8%2C42027%2C1731640420387.1731640422194 2024-11-15T03:13:47,878 INFO [RS_LOG_REPLAY_OPS-regionserver/91f82191efb8:0-1 {event_type=RS_LOG_REPLAY, pid=66}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/WALs/91f82191efb8,42027,1731640420387-splitting/91f82191efb8%2C42027%2C1731640420387.1731640422194 after 1ms 2024-11-15T03:13:47,881 INFO [RS_LOG_REPLAY_OPS-regionserver/91f82191efb8:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(310): Open hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/WALs/91f82191efb8,42027,1731640420387-splitting/91f82191efb8%2C42027%2C1731640420387.1731640422194 took 3ms 2024-11-15T03:13:47,884 DEBUG [RS_LOG_REPLAY_OPS-regionserver/91f82191efb8:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 1b8f1ea215e8c2f78186b80dafed1b28: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-15T03:13:47,886 DEBUG [RS_LOG_REPLAY_OPS-regionserver/91f82191efb8:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 23f804d67a47ba185ffd6f6bc7b75464: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-15T03:13:47,888 DEBUG [RS_LOG_REPLAY_OPS-regionserver/91f82191efb8:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 1beb579fcf8e161a0b9c2051c671a8f2: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-15T03:13:47,890 DEBUG [RS_LOG_REPLAY_OPS-regionserver/91f82191efb8:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for bb2ead3b463edc0f2b603da7d19f1201: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-15T03:13:47,893 DEBUG [RS_LOG_REPLAY_OPS-regionserver/91f82191efb8:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for e0bce1a97a9c7f320b315db443b47c23: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-15T03:13:47,895 DEBUG [RS_LOG_REPLAY_OPS-regionserver/91f82191efb8:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for a73d2d096e2f14a7e9365ac762927d55: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-15T03:13:47,897 DEBUG [RS_LOG_REPLAY_OPS-regionserver/91f82191efb8:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for d0eff1b91bc9b77579a5c44f1c985985: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-15T03:13:47,899 DEBUG [RS_LOG_REPLAY_OPS-regionserver/91f82191efb8:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for dc93af816bdf94cf920a5adc31356783: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-15T03:13:47,901 DEBUG [RS_LOG_REPLAY_OPS-regionserver/91f82191efb8:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 5971f307349447e7f24b0dd05b96cffc: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-15T03:13:47,901 DEBUG [RS_LOG_REPLAY_OPS-regionserver/91f82191efb8:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/WALs/91f82191efb8,42027,1731640420387-splitting/91f82191efb8%2C42027%2C1731640420387.1731640422194 so closing down 2024-11-15T03:13:47,901 DEBUG [RS_LOG_REPLAY_OPS-regionserver/91f82191efb8:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-15T03:13:47,902 INFO [RS_LOG_REPLAY_OPS-regionserver/91f82191efb8:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.OutputSink(145): 3 split writer threads finished 2024-11-15T03:13:47,902 INFO [RS_LOG_REPLAY_OPS-regionserver/91f82191efb8:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(425): Processed 10 edits across 0 Regions in 21 ms; skipped=10; WAL=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/WALs/91f82191efb8,42027,1731640420387-splitting/91f82191efb8%2C42027%2C1731640420387.1731640422194, size=2.7 K, length=2774, corrupted=false, cancelled=false 2024-11-15T03:13:47,902 DEBUG [RS_LOG_REPLAY_OPS-regionserver/91f82191efb8:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(428): Completed split of hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/WALs/91f82191efb8,42027,1731640420387-splitting/91f82191efb8%2C42027%2C1731640420387.1731640422194, journal: Splitting hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/WALs/91f82191efb8,42027,1731640420387-splitting/91f82191efb8%2C42027%2C1731640420387.1731640422194, size=2.7 K (2774bytes) at 1731640427877Finishing writing output for hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/WALs/91f82191efb8,42027,1731640420387-splitting/91f82191efb8%2C42027%2C1731640420387.1731640422194 so closing down at 1731640427901 (+24 ms)3 split writer threads finished at 1731640427902 (+1 ms)Processed 10 edits across 0 Regions in 21 ms; skipped=10; WAL=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/WALs/91f82191efb8,42027,1731640420387-splitting/91f82191efb8%2C42027%2C1731640420387.1731640422194, size=2.7 K, length=2774, corrupted=false, cancelled=false at 1731640427902 2024-11-15T03:13:47,902 DEBUG [RS_LOG_REPLAY_OPS-regionserver/91f82191efb8:0-1 {event_type=RS_LOG_REPLAY, pid=66}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/WALs/91f82191efb8,42027,1731640420387-splitting/91f82191efb8%2C42027%2C1731640420387.1731640422194 2024-11-15T03:13:47,902 DEBUG [RS_LOG_REPLAY_OPS-regionserver/91f82191efb8:0-1 {event_type=RS_LOG_REPLAY, pid=66}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=66 2024-11-15T03:13:47,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42051 {}] master.HMaster(4169): Remote procedure done, pid=66 2024-11-15T03:13:47,907 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/WALs/91f82191efb8,42027,1731640420387-splitting/91f82191efb8%2C42027%2C1731640420387.1731640422194 to hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/oldWALs 2024-11-15T03:13:47,910 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=66, resume processing ppid=65 2024-11-15T03:13:47,910 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 in 201 msec 2024-11-15T03:13:47,912 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:47,915 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=60 2024-11-15T03:13:47,915 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 in 209 msec 2024-11-15T03:13:47,917 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/WALs/91f82191efb8,42027,1731640420387-splitting dir is empty, no logs to split. 2024-11-15T03:13:47,917 INFO [PEWorker-1 {}] master.SplitWALManager(105): 91f82191efb8,42027,1731640420387 WAL count=0, meta=false 2024-11-15T03:13:47,917 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 91f82191efb8,42027,1731640420387 WAL splitting is done? wals=0, meta=false 2024-11-15T03:13:47,920 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN}, {pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN}, {pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN}, {pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1beb579fcf8e161a0b9c2051c671a8f2, ASSIGN}, {pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=bb2ead3b463edc0f2b603da7d19f1201, ASSIGN}, {pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1b8f1ea215e8c2f78186b80dafed1b28, ASSIGN}, {pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d0eff1b91bc9b77579a5c44f1c985985, ASSIGN}, {pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=23f804d67a47ba185ffd6f6bc7b75464, ASSIGN}] 2024-11-15T03:13:47,922 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=23f804d67a47ba185ffd6f6bc7b75464, ASSIGN 2024-11-15T03:13:47,923 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=23f804d67a47ba185ffd6f6bc7b75464, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-15T03:13:47,924 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d0eff1b91bc9b77579a5c44f1c985985, ASSIGN 2024-11-15T03:13:47,924 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d0eff1b91bc9b77579a5c44f1c985985, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-15T03:13:47,925 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1b8f1ea215e8c2f78186b80dafed1b28, ASSIGN 2024-11-15T03:13:47,926 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=1b8f1ea215e8c2f78186b80dafed1b28, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-15T03:13:47,927 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=bb2ead3b463edc0f2b603da7d19f1201, ASSIGN 2024-11-15T03:13:47,928 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=bb2ead3b463edc0f2b603da7d19f1201, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-15T03:13:47,929 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1beb579fcf8e161a0b9c2051c671a8f2, ASSIGN 2024-11-15T03:13:47,930 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=1beb579fcf8e161a0b9c2051c671a8f2, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-15T03:13:47,931 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN 2024-11-15T03:13:47,932 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-15T03:13:47,933 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN 2024-11-15T03:13:47,933 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-15T03:13:47,934 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN 2024-11-15T03:13:47,935 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-15T03:13:48,074 DEBUG [91f82191efb8:42051 {}] balancer.BalancerClusterState(204): Hosts are {91f82191efb8=0} racks are {/default-rack=0} 2024-11-15T03:13:48,074 DEBUG [91f82191efb8:42051 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-15T03:13:48,074 DEBUG [91f82191efb8:42051 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-15T03:13:48,074 DEBUG [91f82191efb8:42051 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-15T03:13:48,075 DEBUG [91f82191efb8:42051 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-15T03:13:48,075 INFO [91f82191efb8:42051 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-15T03:13:48,075 INFO [91f82191efb8:42051 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-15T03:13:48,075 DEBUG [91f82191efb8:42051 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-15T03:13:48,076 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=72 updating hbase:meta row=1b8f1ea215e8c2f78186b80dafed1b28, regionState=OPENING, regionLocation=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,077 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=a73d2d096e2f14a7e9365ac762927d55, regionState=OPENING, regionLocation=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,079 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=73 updating hbase:meta row=d0eff1b91bc9b77579a5c44f1c985985, regionState=OPENING, regionLocation=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,079 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 91f82191efb8:42027 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 91f82191efb8/172.17.0.2:42027 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T03:13:48,081 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=5971f307349447e7f24b0dd05b96cffc, regionState=OPENING, regionLocation=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,082 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=e0bce1a97a9c7f320b315db443b47c23, regionState=OPENING, regionLocation=91f82191efb8,41451,1731640420638 2024-11-15T03:13:48,083 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=91f82191efb8,42027,1731640420387, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=91f82191efb8,42027,1731640420387, seqNum=-1, error=java.net.ConnectException: Call to address=91f82191efb8:42027 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 91f82191efb8/172.17.0.2:42027 2024-11-15T03:13:48,083 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=74 updating hbase:meta row=23f804d67a47ba185ffd6f6bc7b75464, regionState=OPENING, regionLocation=91f82191efb8,41451,1731640420638 2024-11-15T03:13:48,084 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=bb2ead3b463edc0f2b603da7d19f1201, regionState=OPENING, regionLocation=91f82191efb8,41451,1731640420638 2024-11-15T03:13:48,084 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=91f82191efb8,42027,1731640420387, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-15T03:13:48,084 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=91f82191efb8,42027,1731640420387, seqNum=-1 from cache 2024-11-15T03:13:48,084 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=1beb579fcf8e161a0b9c2051c671a8f2, regionState=OPENING, regionLocation=91f82191efb8,41451,1731640420638 2024-11-15T03:13:48,085 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T03:13:48,085 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=91f82191efb8,37523,1731640420593, seqNum=-1] 2024-11-15T03:13:48,085 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 91f82191efb8:42027 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 91f82191efb8/172.17.0.2:42027 2024-11-15T03:13:48,085 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T03:13:48,086 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 91f82191efb8:42027 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 91f82191efb8/172.17.0.2:42027 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T03:13:48,086 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=91f82191efb8,42027,1731640420387, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=91f82191efb8,37523,1731640420593, seqNum=-1, error=java.net.ConnectException: Call to address=91f82191efb8:42027 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 91f82191efb8/172.17.0.2:42027 2024-11-15T03:13:48,086 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=91f82191efb8,42027,1731640420387, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=91f82191efb8,37523,1731640420593, seqNum=-1, error=java.net.ConnectException: Call to address=91f82191efb8:42027 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 91f82191efb8/172.17.0.2:42027 2024-11-15T03:13:48,086 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=91f82191efb8,42027,1731640420387, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=91f82191efb8,37523,1731640420593, seqNum=-1, error=java.net.ConnectException: Call to address=91f82191efb8:42027 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 91f82191efb8/172.17.0.2:42027 2024-11-15T03:13:48,087 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=91f82191efb8,42027,1731640420387, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=91f82191efb8,37523,1731640420593, seqNum=-1, error=java.net.ConnectException: Call to address=91f82191efb8:42027 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 91f82191efb8/172.17.0.2:42027 2024-11-15T03:13:48,087 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=91f82191efb8,42027,1731640420387, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=91f82191efb8,37523,1731640420593, seqNum=-1, error=java.net.ConnectException: Call to address=91f82191efb8:42027 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 91f82191efb8/172.17.0.2:42027 2024-11-15T03:13:48,087 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53789, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T03:13:48,087 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=91f82191efb8,42027,1731640420387, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=91f82191efb8,37523,1731640420593, seqNum=-1, error=java.net.ConnectException: Call to address=91f82191efb8:42027 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 91f82191efb8/172.17.0.2:42027 2024-11-15T03:13:48,087 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 91f82191efb8:42027 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 91f82191efb8/172.17.0.2:42027 2024-11-15T03:13:48,089 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=1beb579fcf8e161a0b9c2051c671a8f2, ASSIGN because future has completed 2024-11-15T03:13:48,090 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1beb579fcf8e161a0b9c2051c671a8f2, server=91f82191efb8,41451,1731640420638}] 2024-11-15T03:13:48,194 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=1b8f1ea215e8c2f78186b80dafed1b28, ASSIGN because future has completed 2024-11-15T03:13:48,194 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1b8f1ea215e8c2f78186b80dafed1b28, server=91f82191efb8,37523,1731640420593}] 2024-11-15T03:13:48,195 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=bb2ead3b463edc0f2b603da7d19f1201, ASSIGN because future has completed 2024-11-15T03:13:48,196 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN because future has completed 2024-11-15T03:13:48,198 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d0eff1b91bc9b77579a5c44f1c985985, ASSIGN because future has completed 2024-11-15T03:13:48,199 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN because future has completed 2024-11-15T03:13:48,199 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure d0eff1b91bc9b77579a5c44f1c985985, server=91f82191efb8,37523,1731640420593}] 2024-11-15T03:13:48,203 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN because future has completed 2024-11-15T03:13:48,204 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=23f804d67a47ba185ffd6f6bc7b75464, ASSIGN because future has completed 2024-11-15T03:13:48,204 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 23f804d67a47ba185ffd6f6bc7b75464, server=91f82191efb8,41451,1731640420638}] 2024-11-15T03:13:48,208 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure e0bce1a97a9c7f320b315db443b47c23, server=91f82191efb8,41451,1731640420638}] 2024-11-15T03:13:48,211 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5971f307349447e7f24b0dd05b96cffc, server=91f82191efb8,37523,1731640420593}] 2024-11-15T03:13:48,215 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure a73d2d096e2f14a7e9365ac762927d55, server=91f82191efb8,37523,1731640420593}] 2024-11-15T03:13:48,219 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638}] 2024-11-15T03:13:48,245 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] handler.AssignRegionHandler(132): Open test,kkk,1731640423069.bb2ead3b463edc0f2b603da7d19f1201. 2024-11-15T03:13:48,246 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7752): Opening region: {ENCODED => bb2ead3b463edc0f2b603da7d19f1201, NAME => 'test,kkk,1731640423069.bb2ead3b463edc0f2b603da7d19f1201.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-11-15T03:13:48,246 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test bb2ead3b463edc0f2b603da7d19f1201 2024-11-15T03:13:48,246 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(898): Instantiated test,kkk,1731640423069.bb2ead3b463edc0f2b603da7d19f1201.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T03:13:48,246 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7794): checking encryption for bb2ead3b463edc0f2b603da7d19f1201 2024-11-15T03:13:48,246 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7797): checking classloading for bb2ead3b463edc0f2b603da7d19f1201 2024-11-15T03:13:48,248 INFO [StoreOpener-bb2ead3b463edc0f2b603da7d19f1201-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region bb2ead3b463edc0f2b603da7d19f1201 2024-11-15T03:13:48,249 INFO [StoreOpener-bb2ead3b463edc0f2b603da7d19f1201-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bb2ead3b463edc0f2b603da7d19f1201 columnFamilyName family 2024-11-15T03:13:48,249 DEBUG [StoreOpener-bb2ead3b463edc0f2b603da7d19f1201-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T03:13:48,250 INFO [StoreOpener-bb2ead3b463edc0f2b603da7d19f1201-1 {}] regionserver.HStore(327): Store=bb2ead3b463edc0f2b603da7d19f1201/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T03:13:48,250 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1038): replaying wal for bb2ead3b463edc0f2b603da7d19f1201 2024-11-15T03:13:48,251 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/bb2ead3b463edc0f2b603da7d19f1201 2024-11-15T03:13:48,253 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/bb2ead3b463edc0f2b603da7d19f1201 2024-11-15T03:13:48,253 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1048): stopping wal replay for bb2ead3b463edc0f2b603da7d19f1201 2024-11-15T03:13:48,254 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1060): Cleaning up temporary data for bb2ead3b463edc0f2b603da7d19f1201 2024-11-15T03:13:48,256 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1093): writing seq id for bb2ead3b463edc0f2b603da7d19f1201 2024-11-15T03:13:48,257 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T03:13:48,257 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1114): Opened bb2ead3b463edc0f2b603da7d19f1201; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74605565, jitterRate=0.11170955002307892}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T03:13:48,257 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bb2ead3b463edc0f2b603da7d19f1201 2024-11-15T03:13:48,257 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1006): Region open journal for bb2ead3b463edc0f2b603da7d19f1201: Running coprocessor pre-open hook at 1731640428247Writing region info on filesystem at 1731640428247Initializing all the Stores at 1731640428248 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731640428248Cleaning up temporary data from old regions at 1731640428254 (+6 ms)Running coprocessor post-open hooks at 1731640428257 (+3 ms)Region opened successfully at 1731640428257 2024-11-15T03:13:48,258 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegionServer(2236): Post open deploy tasks for test,kkk,1731640423069.bb2ead3b463edc0f2b603da7d19f1201., pid=82, masterSystemTime=1731640428242 2024-11-15T03:13:48,261 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegionServer(2266): Finished post open deploy task for test,kkk,1731640423069.bb2ead3b463edc0f2b603da7d19f1201. 2024-11-15T03:13:48,261 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] handler.AssignRegionHandler(153): Opened test,kkk,1731640423069.bb2ead3b463edc0f2b603da7d19f1201. 2024-11-15T03:13:48,261 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(132): Open test,qqq,1731640423069.23f804d67a47ba185ffd6f6bc7b75464. 2024-11-15T03:13:48,261 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7752): Opening region: {ENCODED => 23f804d67a47ba185ffd6f6bc7b75464, NAME => 'test,qqq,1731640423069.23f804d67a47ba185ffd6f6bc7b75464.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-11-15T03:13:48,262 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 23f804d67a47ba185ffd6f6bc7b75464 2024-11-15T03:13:48,262 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(898): Instantiated test,qqq,1731640423069.23f804d67a47ba185ffd6f6bc7b75464.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T03:13:48,262 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7794): checking encryption for 23f804d67a47ba185ffd6f6bc7b75464 2024-11-15T03:13:48,262 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7797): checking classloading for 23f804d67a47ba185ffd6f6bc7b75464 2024-11-15T03:13:48,264 WARN [PEWorker-1 {}] procedure2.ProcedureTestingUtility(201): Set Kill before store update in rollback to: true 2024-11-15T03:13:48,264 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638 2024-11-15T03:13:48,264 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1844): CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638 java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T03:13:48,267 INFO [StoreOpener-23f804d67a47ba185ffd6f6bc7b75464-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 23f804d67a47ba185ffd6f6bc7b75464 2024-11-15T03:13:48,268 INFO [StoreOpener-23f804d67a47ba185ffd6f6bc7b75464-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 23f804d67a47ba185ffd6f6bc7b75464 columnFamilyName family 2024-11-15T03:13:48,269 DEBUG [StoreOpener-23f804d67a47ba185ffd6f6bc7b75464-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T03:13:48,269 INFO [StoreOpener-23f804d67a47ba185ffd6f6bc7b75464-1 {}] regionserver.HStore(327): Store=23f804d67a47ba185ffd6f6bc7b75464/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T03:13:48,270 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1038): replaying wal for 23f804d67a47ba185ffd6f6bc7b75464 2024-11-15T03:13:48,271 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/23f804d67a47ba185ffd6f6bc7b75464 2024-11-15T03:13:48,273 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/23f804d67a47ba185ffd6f6bc7b75464 2024-11-15T03:13:48,273 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1048): stopping wal replay for 23f804d67a47ba185ffd6f6bc7b75464 2024-11-15T03:13:48,274 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1060): Cleaning up temporary data for 23f804d67a47ba185ffd6f6bc7b75464 2024-11-15T03:13:48,276 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1093): writing seq id for 23f804d67a47ba185ffd6f6bc7b75464 2024-11-15T03:13:48,275 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=true, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1845) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 4 more 2024-11-15T03:13:48,278 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-15T03:13:48,278 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=82, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638 2024-11-15T03:13:48,278 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T03:13:48,278 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1114): Opened 23f804d67a47ba185ffd6f6bc7b75464; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60615110, jitterRate=-0.096764475107193}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T03:13:48,278 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 23f804d67a47ba185ffd6f6bc7b75464 2024-11-15T03:13:48,278 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=82, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T03:13:48,279 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1006): Region open journal for 23f804d67a47ba185ffd6f6bc7b75464: Running coprocessor pre-open hook at 1731640428262Writing region info on filesystem at 1731640428262Initializing all the Stores at 1731640428264 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731640428264Cleaning up temporary data from old regions at 1731640428274 (+10 ms)Running coprocessor post-open hooks at 1731640428278 (+4 ms)Region opened successfully at 1731640428278 2024-11-15T03:13:48,280 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2236): Post open deploy tasks for test,qqq,1731640423069.23f804d67a47ba185ffd6f6bc7b75464., pid=78, masterSystemTime=1731640428242 2024-11-15T03:13:48,283 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2266): Finished post open deploy task for test,qqq,1731640423069.23f804d67a47ba185ffd6f6bc7b75464. 2024-11-15T03:13:48,283 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(153): Opened test,qqq,1731640423069.23f804d67a47ba185ffd6f6bc7b75464. 2024-11-15T03:13:48,284 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(132): Open test,ddd,1731640423069.1beb579fcf8e161a0b9c2051c671a8f2. 2024-11-15T03:13:48,284 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7752): Opening region: {ENCODED => 1beb579fcf8e161a0b9c2051c671a8f2, NAME => 'test,ddd,1731640423069.1beb579fcf8e161a0b9c2051c671a8f2.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-11-15T03:13:48,284 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 1beb579fcf8e161a0b9c2051c671a8f2 2024-11-15T03:13:48,284 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(898): Instantiated test,ddd,1731640423069.1beb579fcf8e161a0b9c2051c671a8f2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T03:13:48,284 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7794): checking encryption for 1beb579fcf8e161a0b9c2051c671a8f2 2024-11-15T03:13:48,284 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7797): checking classloading for 1beb579fcf8e161a0b9c2051c671a8f2 2024-11-15T03:13:48,287 INFO [StoreOpener-1beb579fcf8e161a0b9c2051c671a8f2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 1beb579fcf8e161a0b9c2051c671a8f2 2024-11-15T03:13:48,289 INFO [StoreOpener-1beb579fcf8e161a0b9c2051c671a8f2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1beb579fcf8e161a0b9c2051c671a8f2 columnFamilyName family 2024-11-15T03:13:48,289 DEBUG [StoreOpener-1beb579fcf8e161a0b9c2051c671a8f2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T03:13:48,290 INFO [StoreOpener-1beb579fcf8e161a0b9c2051c671a8f2-1 {}] regionserver.HStore(327): Store=1beb579fcf8e161a0b9c2051c671a8f2/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T03:13:48,290 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1038): replaying wal for 1beb579fcf8e161a0b9c2051c671a8f2 2024-11-15T03:13:48,291 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/1beb579fcf8e161a0b9c2051c671a8f2 2024-11-15T03:13:48,293 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/1beb579fcf8e161a0b9c2051c671a8f2 2024-11-15T03:13:48,294 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1048): stopping wal replay for 1beb579fcf8e161a0b9c2051c671a8f2 2024-11-15T03:13:48,294 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1060): Cleaning up temporary data for 1beb579fcf8e161a0b9c2051c671a8f2 2024-11-15T03:13:48,297 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): writing seq id for 1beb579fcf8e161a0b9c2051c671a8f2 2024-11-15T03:13:48,298 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1114): Opened 1beb579fcf8e161a0b9c2051c671a8f2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68041235, jitterRate=0.013893410563468933}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T03:13:48,299 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1beb579fcf8e161a0b9c2051c671a8f2 2024-11-15T03:13:48,299 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1006): Region open journal for 1beb579fcf8e161a0b9c2051c671a8f2: Running coprocessor pre-open hook at 1731640428285Writing region info on filesystem at 1731640428285Initializing all the Stores at 1731640428287 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731640428287Cleaning up temporary data from old regions at 1731640428294 (+7 ms)Running coprocessor post-open hooks at 1731640428299 (+5 ms)Region opened successfully at 1731640428299 2024-11-15T03:13:48,300 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ddd,1731640423069.1beb579fcf8e161a0b9c2051c671a8f2., pid=75, masterSystemTime=1731640428242 2024-11-15T03:13:48,303 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ddd,1731640423069.1beb579fcf8e161a0b9c2051c671a8f2. 2024-11-15T03:13:48,303 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(153): Opened test,ddd,1731640423069.1beb579fcf8e161a0b9c2051c671a8f2. 2024-11-15T03:13:48,303 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(132): Open test,yyy,1731640423069.e0bce1a97a9c7f320b315db443b47c23. 2024-11-15T03:13:48,303 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7752): Opening region: {ENCODED => e0bce1a97a9c7f320b315db443b47c23, NAME => 'test,yyy,1731640423069.e0bce1a97a9c7f320b315db443b47c23.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-15T03:13:48,304 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test e0bce1a97a9c7f320b315db443b47c23 2024-11-15T03:13:48,304 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(898): Instantiated test,yyy,1731640423069.e0bce1a97a9c7f320b315db443b47c23.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T03:13:48,304 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7794): checking encryption for e0bce1a97a9c7f320b315db443b47c23 2024-11-15T03:13:48,304 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7797): checking classloading for e0bce1a97a9c7f320b315db443b47c23 2024-11-15T03:13:48,306 INFO [StoreOpener-e0bce1a97a9c7f320b315db443b47c23-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region e0bce1a97a9c7f320b315db443b47c23 2024-11-15T03:13:48,307 INFO [StoreOpener-e0bce1a97a9c7f320b315db443b47c23-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e0bce1a97a9c7f320b315db443b47c23 columnFamilyName family 2024-11-15T03:13:48,307 DEBUG [StoreOpener-e0bce1a97a9c7f320b315db443b47c23-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T03:13:48,308 INFO [StoreOpener-e0bce1a97a9c7f320b315db443b47c23-1 {}] regionserver.HStore(327): Store=e0bce1a97a9c7f320b315db443b47c23/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T03:13:48,308 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1038): replaying wal for e0bce1a97a9c7f320b315db443b47c23 2024-11-15T03:13:48,309 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/e0bce1a97a9c7f320b315db443b47c23 2024-11-15T03:13:48,311 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/e0bce1a97a9c7f320b315db443b47c23 2024-11-15T03:13:48,311 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1048): stopping wal replay for e0bce1a97a9c7f320b315db443b47c23 2024-11-15T03:13:48,312 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1060): Cleaning up temporary data for e0bce1a97a9c7f320b315db443b47c23 2024-11-15T03:13:48,314 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1093): writing seq id for e0bce1a97a9c7f320b315db443b47c23 2024-11-15T03:13:48,316 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1114): Opened e0bce1a97a9c7f320b315db443b47c23; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68046164, jitterRate=0.013966858386993408}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T03:13:48,316 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e0bce1a97a9c7f320b315db443b47c23 2024-11-15T03:13:48,316 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1006): Region open journal for e0bce1a97a9c7f320b315db443b47c23: Running coprocessor pre-open hook at 1731640428304Writing region info on filesystem at 1731640428304Initializing all the Stores at 1731640428305 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731640428305Cleaning up temporary data from old regions at 1731640428312 (+7 ms)Running coprocessor post-open hooks at 1731640428316 (+4 ms)Region opened successfully at 1731640428316 2024-11-15T03:13:48,317 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1731640423069.e0bce1a97a9c7f320b315db443b47c23., pid=79, masterSystemTime=1731640428242 2024-11-15T03:13:48,320 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1731640423069.e0bce1a97a9c7f320b315db443b47c23. 2024-11-15T03:13:48,320 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(153): Opened test,yyy,1731640423069.e0bce1a97a9c7f320b315db443b47c23. 2024-11-15T03:13:48,324 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test' 2024-11-15T03:13:48,352 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(132): Open test,ccc,1731640423069.5971f307349447e7f24b0dd05b96cffc. 2024-11-15T03:13:48,352 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7752): Opening region: {ENCODED => 5971f307349447e7f24b0dd05b96cffc, NAME => 'test,ccc,1731640423069.5971f307349447e7f24b0dd05b96cffc.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-15T03:13:48,352 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 5971f307349447e7f24b0dd05b96cffc 2024-11-15T03:13:48,352 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(898): Instantiated test,ccc,1731640423069.5971f307349447e7f24b0dd05b96cffc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T03:13:48,353 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7794): checking encryption for 5971f307349447e7f24b0dd05b96cffc 2024-11-15T03:13:48,353 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7797): checking classloading for 5971f307349447e7f24b0dd05b96cffc 2024-11-15T03:13:48,354 INFO [StoreOpener-5971f307349447e7f24b0dd05b96cffc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 5971f307349447e7f24b0dd05b96cffc 2024-11-15T03:13:48,355 INFO [StoreOpener-5971f307349447e7f24b0dd05b96cffc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5971f307349447e7f24b0dd05b96cffc columnFamilyName family 2024-11-15T03:13:48,355 DEBUG [StoreOpener-5971f307349447e7f24b0dd05b96cffc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T03:13:48,356 INFO [StoreOpener-5971f307349447e7f24b0dd05b96cffc-1 {}] regionserver.HStore(327): Store=5971f307349447e7f24b0dd05b96cffc/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T03:13:48,356 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1038): replaying wal for 5971f307349447e7f24b0dd05b96cffc 2024-11-15T03:13:48,357 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/5971f307349447e7f24b0dd05b96cffc 2024-11-15T03:13:48,358 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/5971f307349447e7f24b0dd05b96cffc 2024-11-15T03:13:48,359 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1048): stopping wal replay for 5971f307349447e7f24b0dd05b96cffc 2024-11-15T03:13:48,359 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1060): Cleaning up temporary data for 5971f307349447e7f24b0dd05b96cffc 2024-11-15T03:13:48,361 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1093): writing seq id for 5971f307349447e7f24b0dd05b96cffc 2024-11-15T03:13:48,362 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1114): Opened 5971f307349447e7f24b0dd05b96cffc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71465335, jitterRate=0.06491647660732269}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T03:13:48,362 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5971f307349447e7f24b0dd05b96cffc 2024-11-15T03:13:48,362 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1006): Region open journal for 5971f307349447e7f24b0dd05b96cffc: Running coprocessor pre-open hook at 1731640428353Writing region info on filesystem at 1731640428353Initializing all the Stores at 1731640428354 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731640428354Cleaning up temporary data from old regions at 1731640428359 (+5 ms)Running coprocessor post-open hooks at 1731640428362 (+3 ms)Region opened successfully at 1731640428362 2024-11-15T03:13:48,363 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1731640423069.5971f307349447e7f24b0dd05b96cffc., pid=80, masterSystemTime=1731640428347 2024-11-15T03:13:48,365 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ccc,1731640423069.5971f307349447e7f24b0dd05b96cffc. 2024-11-15T03:13:48,365 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(153): Opened test,ccc,1731640423069.5971f307349447e7f24b0dd05b96cffc. 2024-11-15T03:13:48,365 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(132): Open test,uuu,1731640423069.1b8f1ea215e8c2f78186b80dafed1b28. 2024-11-15T03:13:48,365 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7752): Opening region: {ENCODED => 1b8f1ea215e8c2f78186b80dafed1b28, NAME => 'test,uuu,1731640423069.1b8f1ea215e8c2f78186b80dafed1b28.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-15T03:13:48,365 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 1b8f1ea215e8c2f78186b80dafed1b28 2024-11-15T03:13:48,366 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(898): Instantiated test,uuu,1731640423069.1b8f1ea215e8c2f78186b80dafed1b28.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T03:13:48,366 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7794): checking encryption for 1b8f1ea215e8c2f78186b80dafed1b28 2024-11-15T03:13:48,366 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7797): checking classloading for 1b8f1ea215e8c2f78186b80dafed1b28 2024-11-15T03:13:48,367 INFO [StoreOpener-1b8f1ea215e8c2f78186b80dafed1b28-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 1b8f1ea215e8c2f78186b80dafed1b28 2024-11-15T03:13:48,368 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T03:13:48,368 INFO [StoreOpener-1b8f1ea215e8c2f78186b80dafed1b28-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1b8f1ea215e8c2f78186b80dafed1b28 columnFamilyName family 2024-11-15T03:13:48,368 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T03:13:48,368 DEBUG [StoreOpener-1b8f1ea215e8c2f78186b80dafed1b28-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T03:13:48,368 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T03:13:48,369 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T03:13:48,369 INFO [StoreOpener-1b8f1ea215e8c2f78186b80dafed1b28-1 {}] regionserver.HStore(327): Store=1b8f1ea215e8c2f78186b80dafed1b28/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T03:13:48,369 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1038): replaying wal for 1b8f1ea215e8c2f78186b80dafed1b28 2024-11-15T03:13:48,370 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T03:13:48,370 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T03:13:48,370 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T03:13:48,370 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/1b8f1ea215e8c2f78186b80dafed1b28 2024-11-15T03:13:48,370 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T03:13:48,371 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-15T03:13:48,371 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/1b8f1ea215e8c2f78186b80dafed1b28 2024-11-15T03:13:48,372 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1048): stopping wal replay for 1b8f1ea215e8c2f78186b80dafed1b28 2024-11-15T03:13:48,372 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1060): Cleaning up temporary data for 1b8f1ea215e8c2f78186b80dafed1b28 2024-11-15T03:13:48,374 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1093): writing seq id for 1b8f1ea215e8c2f78186b80dafed1b28 2024-11-15T03:13:48,375 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1114): Opened 1b8f1ea215e8c2f78186b80dafed1b28; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70444277, jitterRate=0.04970152676105499}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T03:13:48,375 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1b8f1ea215e8c2f78186b80dafed1b28 2024-11-15T03:13:48,375 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1006): Region open journal for 1b8f1ea215e8c2f78186b80dafed1b28: Running coprocessor pre-open hook at 1731640428366Writing region info on filesystem at 1731640428366Initializing all the Stores at 1731640428367 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731640428367Cleaning up temporary data from old regions at 1731640428372 (+5 ms)Running coprocessor post-open hooks at 1731640428375 (+3 ms)Region opened successfully at 1731640428375 2024-11-15T03:13:48,376 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2236): Post open deploy tasks for test,uuu,1731640423069.1b8f1ea215e8c2f78186b80dafed1b28., pid=76, masterSystemTime=1731640428347 2024-11-15T03:13:48,377 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42051 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T03:13:48,378 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-15T03:13:48,378 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T03:13:48,378 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T03:13:48,378 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,378 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T03:13:48,378 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,378 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T03:13:48,378 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,379 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=81, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T03:13:48,379 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,379 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T03:13:48,379 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,379 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=82, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T03:13:48,379 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,379 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T03:13:48,379 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,379 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T03:13:48,379 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,379 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T03:13:48,379 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T03:13:48,379 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T03:13:48,379 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,379 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T03:13:48,380 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T03:13:48,382 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-15T03:13:48,382 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=dc93af816bdf94cf920a5adc31356783, REOPEN/MOVE 2024-11-15T03:13:48,385 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42051 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T03:13:48,388 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true 2024-11-15T03:13:48,388 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,388 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,388 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T03:13:48,388 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42051 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T03:13:48,388 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,388 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,388 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,388 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN 2024-11-15T03:13:48,388 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN 2024-11-15T03:13:48,388 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN 2024-11-15T03:13:48,389 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1beb579fcf8e161a0b9c2051c671a8f2, ASSIGN 2024-11-15T03:13:48,389 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb2ead3b463edc0f2b603da7d19f1201, ASSIGN 2024-11-15T03:13:48,389 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b8f1ea215e8c2f78186b80dafed1b28, ASSIGN 2024-11-15T03:13:48,389 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d0eff1b91bc9b77579a5c44f1c985985, ASSIGN 2024-11-15T03:13:48,389 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23f804d67a47ba185ffd6f6bc7b75464, ASSIGN 2024-11-15T03:13:48,389 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1beb579fcf8e161a0b9c2051c671a8f2, server=91f82191efb8,41451,1731640420638 2024-11-15T03:13:48,389 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1b8f1ea215e8c2f78186b80dafed1b28, server=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,389 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure d0eff1b91bc9b77579a5c44f1c985985, server=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,389 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 23f804d67a47ba185ffd6f6bc7b75464, server=91f82191efb8,41451,1731640420638 2024-11-15T03:13:48,389 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure e0bce1a97a9c7f320b315db443b47c23, server=91f82191efb8,41451,1731640420638 2024-11-15T03:13:48,389 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5971f307349447e7f24b0dd05b96cffc, server=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,389 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=81, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure a73d2d096e2f14a7e9365ac762927d55, server=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,389 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=82, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638 2024-11-15T03:13:48,389 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,389 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,389 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,389 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,390 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,390 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,390 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,390 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,390 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN 2024-11-15T03:13:48,390 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,390 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN 2024-11-15T03:13:48,390 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,390 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN 2024-11-15T03:13:48,390 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1beb579fcf8e161a0b9c2051c671a8f2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,390 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1beb579fcf8e161a0b9c2051c671a8f2, ASSIGN 2024-11-15T03:13:48,390 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb2ead3b463edc0f2b603da7d19f1201, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,390 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb2ead3b463edc0f2b603da7d19f1201, ASSIGN 2024-11-15T03:13:48,390 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b8f1ea215e8c2f78186b80dafed1b28, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,390 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b8f1ea215e8c2f78186b80dafed1b28, ASSIGN 2024-11-15T03:13:48,390 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d0eff1b91bc9b77579a5c44f1c985985, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,390 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d0eff1b91bc9b77579a5c44f1c985985, ASSIGN 2024-11-15T03:13:48,391 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23f804d67a47ba185ffd6f6bc7b75464, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,391 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23f804d67a47ba185ffd6f6bc7b75464, ASSIGN 2024-11-15T03:13:48,391 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1beb579fcf8e161a0b9c2051c671a8f2, server=91f82191efb8,41451,1731640420638 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,391 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1b8f1ea215e8c2f78186b80dafed1b28, server=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,391 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure d0eff1b91bc9b77579a5c44f1c985985, server=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,391 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 23f804d67a47ba185ffd6f6bc7b75464, server=91f82191efb8,41451,1731640420638 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,391 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure e0bce1a97a9c7f320b315db443b47c23, server=91f82191efb8,41451,1731640420638 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,391 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5971f307349447e7f24b0dd05b96cffc, server=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,391 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=81, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure a73d2d096e2f14a7e9365ac762927d55, server=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,391 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=82, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,392 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 20 msec 2024-11-15T03:13:48,394 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=65 2024-11-15T03:13:48,394 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=91f82191efb8,37523,1731640420593, table=hbase:meta, region=1588230740 2024-11-15T03:13:48,394 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 91f82191efb8,37523,1731640420593, state=OPEN 2024-11-15T03:13:48,402 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "91f82191efb8" port: 37523 start_code: 1731640420593 } transition { transition_code: OPENED region_info { region_id: 1731640423069 table_name { namespace: "default" qualifier: "test" } start_key: "uuu" end_key: "vvv" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 76 initiating_master_active_time: 1731640420749 }; retry (#0) after 1005ms delay (Master is coming online...). org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-15T03:13:48,462 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41451-0x1013c7bdd830003, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:48,462 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37523-0x1013c7bdd830002, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:48,462 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42051-0x1013c7bdd830000, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:48,462 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:48,462 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:48,462 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:48,464 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN to state=OFFLINE, location=null, table=test, region=5971f307349447e7f24b0dd05b96cffc to restore RIT 2024-11-15T03:13:48,464 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN to state=OFFLINE, location=null, table=test, region=e0bce1a97a9c7f320b315db443b47c23 to restore RIT 2024-11-15T03:13:48,464 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN to state=OFFLINE, location=null, table=test, region=a73d2d096e2f14a7e9365ac762927d55 to restore RIT 2024-11-15T03:13:48,464 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1beb579fcf8e161a0b9c2051c671a8f2, ASSIGN to state=OFFLINE, location=null, table=test, region=1beb579fcf8e161a0b9c2051c671a8f2 to restore RIT 2024-11-15T03:13:48,464 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb2ead3b463edc0f2b603da7d19f1201, ASSIGN to state=OFFLINE, location=null, table=test, region=bb2ead3b463edc0f2b603da7d19f1201 to restore RIT 2024-11-15T03:13:48,464 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b8f1ea215e8c2f78186b80dafed1b28, ASSIGN to state=OFFLINE, location=null, table=test, region=1b8f1ea215e8c2f78186b80dafed1b28 to restore RIT 2024-11-15T03:13:48,464 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d0eff1b91bc9b77579a5c44f1c985985, ASSIGN to state=OFFLINE, location=null, table=test, region=d0eff1b91bc9b77579a5c44f1c985985 to restore RIT 2024-11-15T03:13:48,464 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23f804d67a47ba185ffd6f6bc7b75464, ASSIGN to state=OFFLINE, location=null, table=test, region=23f804d67a47ba185ffd6f6bc7b75464 to restore RIT 2024-11-15T03:13:48,465 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731640458465 2024-11-15T03:13:48,465 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-15T03:13:48,465 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T03:13:48,468 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-15T03:13:48,470 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-15T03:13:48,470 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=81, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure a73d2d096e2f14a7e9365ac762927d55, server=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,470 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T03:13:48,470 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=81, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure a73d2d096e2f14a7e9365ac762927d55, server=91f82191efb8,37523,1731640420593 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T03:13:48,498 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5965b596ce7b87fabde47569d60c3f95, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,499 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6130297c45bd23340b368de2cc708c90, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,499 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5971f307349447e7f24b0dd05b96cffc, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,499 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1beb579fcf8e161a0b9c2051c671a8f2, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,500 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0f9e1e28ca8906f0f82f959a60c55ce, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,500 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4b611da4a73d4cfc99379713486e21c, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,500 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0eff1b91bc9b77579a5c44f1c985985, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,500 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48ff514f5498660a4e8fc753340bb847, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,501 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5219cfe492c74eb8312930d328964b6, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,501 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1704275babe02e36b29cfaadf5b81af, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,501 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb2ead3b463edc0f2b603da7d19f1201, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,501 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=860df8edb65f2fdd3c8bc00f89f94ba0, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,502 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eaf85cc654192e46d3ea0904493eea12, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,502 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a73d2d096e2f14a7e9365ac762927d55, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,502 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b96380a8d7110dffa8cd7844252025dd, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,502 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fac2a1b464a5ff2d581af6baef327bde, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,502 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f804d67a47ba185ffd6f6bc7b75464, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,503 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc930ecfca79abbf57ed28c20f79230f, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,503 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ef03d91ad777a77cf66f510e28419ba, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,503 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=21df709a703798d68290a0a63322957f, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,503 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1b8f1ea215e8c2f78186b80dafed1b28, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,503 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc93af816bdf94cf920a5adc31356783, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=5 2024-11-15T03:13:48,503 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=155ef591549501ad2da24d2c559fb7f9, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,504 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6cca2d3f6a001d96f34b7a93afec8fe7, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,504 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0bce1a97a9c7f320b315db443b47c23, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,504 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17e8b3b889a606b8c6826552e784d401, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,504 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-15T03:13:48,504 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731640488504 2024-11-15T03:13:48,504 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731640548504 2024-11-15T03:13:48,504 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 38 msec 2024-11-15T03:13:48,504 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T03:13:48,504 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T03:13:48,504 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T03:13:48,504 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T03:13:48,505 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T03:13:48,505 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T03:13:48,505 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T03:13:48,505 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T03:13:48,506 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-15T03:13:48,509 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-15T03:13:48,509 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T03:13:48,510 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T03:13:48,510 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,510 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T03:13:48,510 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,510 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T03:13:48,510 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,510 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=81, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T03:13:48,510 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,510 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T03:13:48,510 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,510 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,510 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T03:13:48,510 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,510 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T03:13:48,510 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,510 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T03:13:48,510 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T03:13:48,510 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T03:13:48,510 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,510 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T03:13:48,510 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T03:13:48,512 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-15T03:13:48,512 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=dc93af816bdf94cf920a5adc31356783, REOPEN/MOVE 2024-11-15T03:13:48,514 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true 2024-11-15T03:13:48,514 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,514 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,514 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T03:13:48,514 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,514 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,514 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,514 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN 2024-11-15T03:13:48,515 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN 2024-11-15T03:13:48,515 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN 2024-11-15T03:13:48,515 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1beb579fcf8e161a0b9c2051c671a8f2, ASSIGN 2024-11-15T03:13:48,515 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb2ead3b463edc0f2b603da7d19f1201, ASSIGN 2024-11-15T03:13:48,515 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b8f1ea215e8c2f78186b80dafed1b28, ASSIGN 2024-11-15T03:13:48,515 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d0eff1b91bc9b77579a5c44f1c985985, ASSIGN 2024-11-15T03:13:48,515 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23f804d67a47ba185ffd6f6bc7b75464, ASSIGN 2024-11-15T03:13:48,515 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1beb579fcf8e161a0b9c2051c671a8f2, server=91f82191efb8,41451,1731640420638 2024-11-15T03:13:48,515 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1b8f1ea215e8c2f78186b80dafed1b28, server=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,515 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure d0eff1b91bc9b77579a5c44f1c985985, server=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,515 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 23f804d67a47ba185ffd6f6bc7b75464, server=91f82191efb8,41451,1731640420638 2024-11-15T03:13:48,515 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure e0bce1a97a9c7f320b315db443b47c23, server=91f82191efb8,41451,1731640420638 2024-11-15T03:13:48,515 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5971f307349447e7f24b0dd05b96cffc, server=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,515 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=81, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure a73d2d096e2f14a7e9365ac762927d55, server=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,515 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,515 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,515 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,515 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,515 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,515 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,515 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,515 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,515 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN 2024-11-15T03:13:48,515 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,515 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN 2024-11-15T03:13:48,515 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,515 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN 2024-11-15T03:13:48,515 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1beb579fcf8e161a0b9c2051c671a8f2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,515 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1beb579fcf8e161a0b9c2051c671a8f2, ASSIGN 2024-11-15T03:13:48,516 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb2ead3b463edc0f2b603da7d19f1201, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,516 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb2ead3b463edc0f2b603da7d19f1201, ASSIGN 2024-11-15T03:13:48,516 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b8f1ea215e8c2f78186b80dafed1b28, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,516 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b8f1ea215e8c2f78186b80dafed1b28, ASSIGN 2024-11-15T03:13:48,516 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d0eff1b91bc9b77579a5c44f1c985985, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,516 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d0eff1b91bc9b77579a5c44f1c985985, ASSIGN 2024-11-15T03:13:48,516 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23f804d67a47ba185ffd6f6bc7b75464, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,516 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23f804d67a47ba185ffd6f6bc7b75464, ASSIGN 2024-11-15T03:13:48,516 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1beb579fcf8e161a0b9c2051c671a8f2, server=91f82191efb8,41451,1731640420638 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,516 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1b8f1ea215e8c2f78186b80dafed1b28, server=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,516 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure d0eff1b91bc9b77579a5c44f1c985985, server=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,516 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 23f804d67a47ba185ffd6f6bc7b75464, server=91f82191efb8,41451,1731640420638 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,516 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure e0bce1a97a9c7f320b315db443b47c23, server=91f82191efb8,41451,1731640420638 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,516 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5971f307349447e7f24b0dd05b96cffc, server=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,516 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=81, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure a73d2d096e2f14a7e9365ac762927d55, server=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,516 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 10 msec 2024-11-15T03:13:48,517 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=65 2024-11-15T03:13:48,517 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=91f82191efb8,37523,1731640420593, table=hbase:meta, region=1588230740 2024-11-15T03:13:48,517 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 91f82191efb8,37523,1731640420593, state=OPEN 2024-11-15T03:13:48,530 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41451-0x1013c7bdd830003, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:48,530 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37523-0x1013c7bdd830002, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:48,530 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42051-0x1013c7bdd830000, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:48,530 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:48,530 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:48,530 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:48,531 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN to state=OFFLINE, location=null, table=test, region=5971f307349447e7f24b0dd05b96cffc to restore RIT 2024-11-15T03:13:48,531 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN to state=OFFLINE, location=null, table=test, region=e0bce1a97a9c7f320b315db443b47c23 to restore RIT 2024-11-15T03:13:48,531 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN to state=OFFLINE, location=null, table=test, region=a73d2d096e2f14a7e9365ac762927d55 to restore RIT 2024-11-15T03:13:48,531 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1beb579fcf8e161a0b9c2051c671a8f2, ASSIGN to state=OFFLINE, location=null, table=test, region=1beb579fcf8e161a0b9c2051c671a8f2 to restore RIT 2024-11-15T03:13:48,531 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb2ead3b463edc0f2b603da7d19f1201, ASSIGN to state=OFFLINE, location=null, table=test, region=bb2ead3b463edc0f2b603da7d19f1201 to restore RIT 2024-11-15T03:13:48,531 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b8f1ea215e8c2f78186b80dafed1b28, ASSIGN to state=OFFLINE, location=null, table=test, region=1b8f1ea215e8c2f78186b80dafed1b28 to restore RIT 2024-11-15T03:13:48,531 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d0eff1b91bc9b77579a5c44f1c985985, ASSIGN to state=OFFLINE, location=null, table=test, region=d0eff1b91bc9b77579a5c44f1c985985 to restore RIT 2024-11-15T03:13:48,531 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23f804d67a47ba185ffd6f6bc7b75464, ASSIGN to state=OFFLINE, location=null, table=test, region=23f804d67a47ba185ffd6f6bc7b75464 to restore RIT 2024-11-15T03:13:48,532 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731640458532 2024-11-15T03:13:48,532 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-15T03:13:48,532 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T03:13:48,533 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-15T03:13:48,540 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-15T03:13:48,540 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=80, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5971f307349447e7f24b0dd05b96cffc, server=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,540 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T03:13:48,540 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=80, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5971f307349447e7f24b0dd05b96cffc, server=91f82191efb8,37523,1731640420593 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T03:13:48,541 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5965b596ce7b87fabde47569d60c3f95, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,541 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6130297c45bd23340b368de2cc708c90, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,541 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5971f307349447e7f24b0dd05b96cffc, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,541 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1beb579fcf8e161a0b9c2051c671a8f2, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,542 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0f9e1e28ca8906f0f82f959a60c55ce, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,542 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4b611da4a73d4cfc99379713486e21c, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,542 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0eff1b91bc9b77579a5c44f1c985985, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,542 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48ff514f5498660a4e8fc753340bb847, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,542 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5219cfe492c74eb8312930d328964b6, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,543 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1704275babe02e36b29cfaadf5b81af, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,543 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb2ead3b463edc0f2b603da7d19f1201, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,543 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=860df8edb65f2fdd3c8bc00f89f94ba0, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,543 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eaf85cc654192e46d3ea0904493eea12, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,544 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a73d2d096e2f14a7e9365ac762927d55, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,544 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b96380a8d7110dffa8cd7844252025dd, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,544 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fac2a1b464a5ff2d581af6baef327bde, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,544 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f804d67a47ba185ffd6f6bc7b75464, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,544 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc930ecfca79abbf57ed28c20f79230f, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,545 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ef03d91ad777a77cf66f510e28419ba, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,545 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=21df709a703798d68290a0a63322957f, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,545 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1b8f1ea215e8c2f78186b80dafed1b28, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,545 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc93af816bdf94cf920a5adc31356783, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=5 2024-11-15T03:13:48,546 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=155ef591549501ad2da24d2c559fb7f9, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,546 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6cca2d3f6a001d96f34b7a93afec8fe7, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,546 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0bce1a97a9c7f320b315db443b47c23, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,546 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17e8b3b889a606b8c6826552e784d401, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,546 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-15T03:13:48,546 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731640488546 2024-11-15T03:13:48,546 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731640548546 2024-11-15T03:13:48,547 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 14 msec 2024-11-15T03:13:48,547 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T03:13:48,547 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T03:13:48,547 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T03:13:48,547 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T03:13:48,547 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T03:13:48,547 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T03:13:48,547 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T03:13:48,548 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T03:13:48,548 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-15T03:13:48,554 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-15T03:13:48,554 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T03:13:48,554 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T03:13:48,554 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,554 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T03:13:48,554 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,554 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T03:13:48,554 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,554 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,554 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T03:13:48,554 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,554 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,554 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T03:13:48,554 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,555 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T03:13:48,555 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,555 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T03:13:48,555 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T03:13:48,555 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T03:13:48,555 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,555 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T03:13:48,555 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T03:13:48,556 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-15T03:13:48,557 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=dc93af816bdf94cf920a5adc31356783, REOPEN/MOVE 2024-11-15T03:13:48,559 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true 2024-11-15T03:13:48,559 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,560 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,560 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T03:13:48,560 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,560 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,560 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,560 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN 2024-11-15T03:13:48,560 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN 2024-11-15T03:13:48,560 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN 2024-11-15T03:13:48,560 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1beb579fcf8e161a0b9c2051c671a8f2, ASSIGN 2024-11-15T03:13:48,560 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb2ead3b463edc0f2b603da7d19f1201, ASSIGN 2024-11-15T03:13:48,560 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b8f1ea215e8c2f78186b80dafed1b28, ASSIGN 2024-11-15T03:13:48,560 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d0eff1b91bc9b77579a5c44f1c985985, ASSIGN 2024-11-15T03:13:48,560 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23f804d67a47ba185ffd6f6bc7b75464, ASSIGN 2024-11-15T03:13:48,560 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1beb579fcf8e161a0b9c2051c671a8f2, server=91f82191efb8,41451,1731640420638 2024-11-15T03:13:48,560 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1b8f1ea215e8c2f78186b80dafed1b28, server=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,560 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure d0eff1b91bc9b77579a5c44f1c985985, server=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,560 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 23f804d67a47ba185ffd6f6bc7b75464, server=91f82191efb8,41451,1731640420638 2024-11-15T03:13:48,560 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure e0bce1a97a9c7f320b315db443b47c23, server=91f82191efb8,41451,1731640420638 2024-11-15T03:13:48,560 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5971f307349447e7f24b0dd05b96cffc, server=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,560 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,561 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,561 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,561 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,561 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,561 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,561 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,561 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,561 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN 2024-11-15T03:13:48,561 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,561 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN 2024-11-15T03:13:48,561 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,561 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN 2024-11-15T03:13:48,561 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1beb579fcf8e161a0b9c2051c671a8f2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,561 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1beb579fcf8e161a0b9c2051c671a8f2, ASSIGN 2024-11-15T03:13:48,561 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb2ead3b463edc0f2b603da7d19f1201, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,561 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb2ead3b463edc0f2b603da7d19f1201, ASSIGN 2024-11-15T03:13:48,561 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b8f1ea215e8c2f78186b80dafed1b28, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,561 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b8f1ea215e8c2f78186b80dafed1b28, ASSIGN 2024-11-15T03:13:48,561 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d0eff1b91bc9b77579a5c44f1c985985, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,561 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d0eff1b91bc9b77579a5c44f1c985985, ASSIGN 2024-11-15T03:13:48,562 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23f804d67a47ba185ffd6f6bc7b75464, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,562 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23f804d67a47ba185ffd6f6bc7b75464, ASSIGN 2024-11-15T03:13:48,562 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1beb579fcf8e161a0b9c2051c671a8f2, server=91f82191efb8,41451,1731640420638 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,562 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1b8f1ea215e8c2f78186b80dafed1b28, server=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,562 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure d0eff1b91bc9b77579a5c44f1c985985, server=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,562 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 23f804d67a47ba185ffd6f6bc7b75464, server=91f82191efb8,41451,1731640420638 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,562 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure e0bce1a97a9c7f320b315db443b47c23, server=91f82191efb8,41451,1731640420638 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,562 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5971f307349447e7f24b0dd05b96cffc, server=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,562 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 13 msec 2024-11-15T03:13:48,563 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=65 2024-11-15T03:13:48,564 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=91f82191efb8,37523,1731640420593, table=hbase:meta, region=1588230740 2024-11-15T03:13:48,564 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 91f82191efb8,37523,1731640420593, state=OPEN 2024-11-15T03:13:48,564 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42051 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T03:13:48,565 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42051 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T03:13:48,572 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41451-0x1013c7bdd830003, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:48,572 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37523-0x1013c7bdd830002, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:48,572 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42051-0x1013c7bdd830000, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:48,572 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:48,572 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:48,572 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:48,573 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN to state=OFFLINE, location=null, table=test, region=5971f307349447e7f24b0dd05b96cffc to restore RIT 2024-11-15T03:13:48,573 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN to state=OFFLINE, location=null, table=test, region=e0bce1a97a9c7f320b315db443b47c23 to restore RIT 2024-11-15T03:13:48,573 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN to state=OFFLINE, location=null, table=test, region=a73d2d096e2f14a7e9365ac762927d55 to restore RIT 2024-11-15T03:13:48,573 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1beb579fcf8e161a0b9c2051c671a8f2, ASSIGN to state=OFFLINE, location=null, table=test, region=1beb579fcf8e161a0b9c2051c671a8f2 to restore RIT 2024-11-15T03:13:48,573 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb2ead3b463edc0f2b603da7d19f1201, ASSIGN to state=OFFLINE, location=null, table=test, region=bb2ead3b463edc0f2b603da7d19f1201 to restore RIT 2024-11-15T03:13:48,573 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b8f1ea215e8c2f78186b80dafed1b28, ASSIGN to state=OFFLINE, location=null, table=test, region=1b8f1ea215e8c2f78186b80dafed1b28 to restore RIT 2024-11-15T03:13:48,573 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d0eff1b91bc9b77579a5c44f1c985985, ASSIGN to state=OFFLINE, location=null, table=test, region=d0eff1b91bc9b77579a5c44f1c985985 to restore RIT 2024-11-15T03:13:48,573 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23f804d67a47ba185ffd6f6bc7b75464, ASSIGN to state=OFFLINE, location=null, table=test, region=23f804d67a47ba185ffd6f6bc7b75464 to restore RIT 2024-11-15T03:13:48,574 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731640458574 2024-11-15T03:13:48,574 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-15T03:13:48,574 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T03:13:48,576 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-15T03:13:48,577 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-15T03:13:48,577 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure e0bce1a97a9c7f320b315db443b47c23, server=91f82191efb8,41451,1731640420638 2024-11-15T03:13:48,577 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T03:13:48,577 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure e0bce1a97a9c7f320b315db443b47c23, server=91f82191efb8,41451,1731640420638 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T03:13:48,581 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5965b596ce7b87fabde47569d60c3f95, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,582 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6130297c45bd23340b368de2cc708c90, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,582 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5971f307349447e7f24b0dd05b96cffc, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,582 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1beb579fcf8e161a0b9c2051c671a8f2, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,582 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0f9e1e28ca8906f0f82f959a60c55ce, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,582 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4b611da4a73d4cfc99379713486e21c, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,582 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0eff1b91bc9b77579a5c44f1c985985, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,582 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48ff514f5498660a4e8fc753340bb847, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,583 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5219cfe492c74eb8312930d328964b6, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,583 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1704275babe02e36b29cfaadf5b81af, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,583 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb2ead3b463edc0f2b603da7d19f1201, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,583 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=860df8edb65f2fdd3c8bc00f89f94ba0, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,583 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eaf85cc654192e46d3ea0904493eea12, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,583 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a73d2d096e2f14a7e9365ac762927d55, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,583 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b96380a8d7110dffa8cd7844252025dd, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,584 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fac2a1b464a5ff2d581af6baef327bde, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,584 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f804d67a47ba185ffd6f6bc7b75464, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,584 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc930ecfca79abbf57ed28c20f79230f, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,584 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ef03d91ad777a77cf66f510e28419ba, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,584 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=21df709a703798d68290a0a63322957f, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,585 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1b8f1ea215e8c2f78186b80dafed1b28, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,585 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc93af816bdf94cf920a5adc31356783, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=5 2024-11-15T03:13:48,585 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=155ef591549501ad2da24d2c559fb7f9, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,585 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6cca2d3f6a001d96f34b7a93afec8fe7, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,585 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0bce1a97a9c7f320b315db443b47c23, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,585 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17e8b3b889a606b8c6826552e784d401, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,585 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-15T03:13:48,585 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731640488585 2024-11-15T03:13:48,585 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731640548585 2024-11-15T03:13:48,585 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 11 msec 2024-11-15T03:13:48,586 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T03:13:48,586 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T03:13:48,586 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T03:13:48,586 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T03:13:48,586 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T03:13:48,586 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T03:13:48,586 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T03:13:48,586 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T03:13:48,587 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-15T03:13:48,590 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-15T03:13:48,590 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T03:13:48,590 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T03:13:48,590 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,590 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,590 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T03:13:48,590 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,590 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,590 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T03:13:48,590 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,590 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,590 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T03:13:48,590 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,591 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T03:13:48,591 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,591 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T03:13:48,591 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T03:13:48,591 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T03:13:48,591 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,591 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T03:13:48,591 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T03:13:48,592 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-15T03:13:48,592 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=dc93af816bdf94cf920a5adc31356783, REOPEN/MOVE 2024-11-15T03:13:48,594 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true 2024-11-15T03:13:48,594 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,594 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,594 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T03:13:48,594 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,594 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,594 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,594 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN 2024-11-15T03:13:48,594 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN 2024-11-15T03:13:48,594 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN 2024-11-15T03:13:48,594 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1beb579fcf8e161a0b9c2051c671a8f2, ASSIGN 2024-11-15T03:13:48,594 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb2ead3b463edc0f2b603da7d19f1201, ASSIGN 2024-11-15T03:13:48,594 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b8f1ea215e8c2f78186b80dafed1b28, ASSIGN 2024-11-15T03:13:48,594 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d0eff1b91bc9b77579a5c44f1c985985, ASSIGN 2024-11-15T03:13:48,594 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23f804d67a47ba185ffd6f6bc7b75464, ASSIGN 2024-11-15T03:13:48,595 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1beb579fcf8e161a0b9c2051c671a8f2, server=91f82191efb8,41451,1731640420638 2024-11-15T03:13:48,595 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1b8f1ea215e8c2f78186b80dafed1b28, server=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,595 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure d0eff1b91bc9b77579a5c44f1c985985, server=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,595 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 23f804d67a47ba185ffd6f6bc7b75464, server=91f82191efb8,41451,1731640420638 2024-11-15T03:13:48,595 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure e0bce1a97a9c7f320b315db443b47c23, server=91f82191efb8,41451,1731640420638 2024-11-15T03:13:48,595 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,595 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,595 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,595 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,595 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,595 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,595 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,595 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,595 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN 2024-11-15T03:13:48,595 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,595 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN 2024-11-15T03:13:48,595 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,595 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN 2024-11-15T03:13:48,595 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1beb579fcf8e161a0b9c2051c671a8f2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,595 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1beb579fcf8e161a0b9c2051c671a8f2, ASSIGN 2024-11-15T03:13:48,595 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb2ead3b463edc0f2b603da7d19f1201, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,595 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb2ead3b463edc0f2b603da7d19f1201, ASSIGN 2024-11-15T03:13:48,595 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b8f1ea215e8c2f78186b80dafed1b28, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,596 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b8f1ea215e8c2f78186b80dafed1b28, ASSIGN 2024-11-15T03:13:48,596 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d0eff1b91bc9b77579a5c44f1c985985, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,596 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d0eff1b91bc9b77579a5c44f1c985985, ASSIGN 2024-11-15T03:13:48,596 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23f804d67a47ba185ffd6f6bc7b75464, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,596 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23f804d67a47ba185ffd6f6bc7b75464, ASSIGN 2024-11-15T03:13:48,596 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1beb579fcf8e161a0b9c2051c671a8f2, server=91f82191efb8,41451,1731640420638 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,596 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1b8f1ea215e8c2f78186b80dafed1b28, server=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,596 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure d0eff1b91bc9b77579a5c44f1c985985, server=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,596 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 23f804d67a47ba185ffd6f6bc7b75464, server=91f82191efb8,41451,1731640420638 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,596 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure e0bce1a97a9c7f320b315db443b47c23, server=91f82191efb8,41451,1731640420638 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,596 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-11-15T03:13:48,597 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=65 2024-11-15T03:13:48,597 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=91f82191efb8,37523,1731640420593, table=hbase:meta, region=1588230740 2024-11-15T03:13:48,597 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 91f82191efb8,37523,1731640420593, state=OPEN 2024-11-15T03:13:48,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41451-0x1013c7bdd830003, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:48,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42051-0x1013c7bdd830000, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:48,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37523-0x1013c7bdd830002, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:48,609 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:48,609 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:48,609 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:48,610 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN to state=OFFLINE, location=null, table=test, region=5971f307349447e7f24b0dd05b96cffc to restore RIT 2024-11-15T03:13:48,610 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN to state=OFFLINE, location=null, table=test, region=e0bce1a97a9c7f320b315db443b47c23 to restore RIT 2024-11-15T03:13:48,610 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN to state=OFFLINE, location=null, table=test, region=a73d2d096e2f14a7e9365ac762927d55 to restore RIT 2024-11-15T03:13:48,610 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1beb579fcf8e161a0b9c2051c671a8f2, ASSIGN to state=OFFLINE, location=null, table=test, region=1beb579fcf8e161a0b9c2051c671a8f2 to restore RIT 2024-11-15T03:13:48,610 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb2ead3b463edc0f2b603da7d19f1201, ASSIGN to state=OFFLINE, location=null, table=test, region=bb2ead3b463edc0f2b603da7d19f1201 to restore RIT 2024-11-15T03:13:48,610 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b8f1ea215e8c2f78186b80dafed1b28, ASSIGN to state=OFFLINE, location=null, table=test, region=1b8f1ea215e8c2f78186b80dafed1b28 to restore RIT 2024-11-15T03:13:48,610 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d0eff1b91bc9b77579a5c44f1c985985, ASSIGN to state=OFFLINE, location=null, table=test, region=d0eff1b91bc9b77579a5c44f1c985985 to restore RIT 2024-11-15T03:13:48,610 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23f804d67a47ba185ffd6f6bc7b75464, ASSIGN to state=OFFLINE, location=null, table=test, region=23f804d67a47ba185ffd6f6bc7b75464 to restore RIT 2024-11-15T03:13:48,611 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731640458611 2024-11-15T03:13:48,611 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-15T03:13:48,611 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T03:13:48,612 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-15T03:13:48,613 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-15T03:13:48,614 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=78, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 23f804d67a47ba185ffd6f6bc7b75464, server=91f82191efb8,41451,1731640420638 2024-11-15T03:13:48,614 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T03:13:48,614 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=78, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 23f804d67a47ba185ffd6f6bc7b75464, server=91f82191efb8,41451,1731640420638 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T03:13:48,617 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5965b596ce7b87fabde47569d60c3f95, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,618 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6130297c45bd23340b368de2cc708c90, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,618 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5971f307349447e7f24b0dd05b96cffc, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,618 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1beb579fcf8e161a0b9c2051c671a8f2, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,618 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0f9e1e28ca8906f0f82f959a60c55ce, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,619 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4b611da4a73d4cfc99379713486e21c, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,619 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0eff1b91bc9b77579a5c44f1c985985, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,619 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48ff514f5498660a4e8fc753340bb847, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,619 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5219cfe492c74eb8312930d328964b6, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,619 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1704275babe02e36b29cfaadf5b81af, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,620 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb2ead3b463edc0f2b603da7d19f1201, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,620 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=860df8edb65f2fdd3c8bc00f89f94ba0, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,620 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eaf85cc654192e46d3ea0904493eea12, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,620 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a73d2d096e2f14a7e9365ac762927d55, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,620 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b96380a8d7110dffa8cd7844252025dd, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,621 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fac2a1b464a5ff2d581af6baef327bde, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,621 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f804d67a47ba185ffd6f6bc7b75464, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,621 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc930ecfca79abbf57ed28c20f79230f, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,621 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ef03d91ad777a77cf66f510e28419ba, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,622 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=21df709a703798d68290a0a63322957f, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,622 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1b8f1ea215e8c2f78186b80dafed1b28, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,622 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc93af816bdf94cf920a5adc31356783, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=5 2024-11-15T03:13:48,622 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=155ef591549501ad2da24d2c559fb7f9, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,622 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6cca2d3f6a001d96f34b7a93afec8fe7, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,623 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0bce1a97a9c7f320b315db443b47c23, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,623 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17e8b3b889a606b8c6826552e784d401, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,623 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-15T03:13:48,623 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731640488623 2024-11-15T03:13:48,623 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731640548623 2024-11-15T03:13:48,623 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 12 msec 2024-11-15T03:13:48,623 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T03:13:48,623 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T03:13:48,623 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T03:13:48,623 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T03:13:48,623 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T03:13:48,623 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T03:13:48,623 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T03:13:48,624 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T03:13:48,624 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-15T03:13:48,628 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-15T03:13:48,628 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T03:13:48,628 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T03:13:48,628 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,628 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,628 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,629 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,629 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T03:13:48,629 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,629 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,629 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T03:13:48,629 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,629 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T03:13:48,629 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,629 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T03:13:48,629 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T03:13:48,629 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T03:13:48,629 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,629 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T03:13:48,629 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T03:13:48,631 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-15T03:13:48,631 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=dc93af816bdf94cf920a5adc31356783, REOPEN/MOVE 2024-11-15T03:13:48,634 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true 2024-11-15T03:13:48,634 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,634 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,634 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T03:13:48,634 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,634 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,634 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,634 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN 2024-11-15T03:13:48,634 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN 2024-11-15T03:13:48,634 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN 2024-11-15T03:13:48,635 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1beb579fcf8e161a0b9c2051c671a8f2, ASSIGN 2024-11-15T03:13:48,635 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb2ead3b463edc0f2b603da7d19f1201, ASSIGN 2024-11-15T03:13:48,635 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b8f1ea215e8c2f78186b80dafed1b28, ASSIGN 2024-11-15T03:13:48,635 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d0eff1b91bc9b77579a5c44f1c985985, ASSIGN 2024-11-15T03:13:48,635 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23f804d67a47ba185ffd6f6bc7b75464, ASSIGN 2024-11-15T03:13:48,635 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1beb579fcf8e161a0b9c2051c671a8f2, server=91f82191efb8,41451,1731640420638 2024-11-15T03:13:48,635 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1b8f1ea215e8c2f78186b80dafed1b28, server=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,635 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure d0eff1b91bc9b77579a5c44f1c985985, server=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,635 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 23f804d67a47ba185ffd6f6bc7b75464, server=91f82191efb8,41451,1731640420638 2024-11-15T03:13:48,635 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,635 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,635 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,635 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,635 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,635 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,635 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,635 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,635 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN 2024-11-15T03:13:48,635 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,635 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN 2024-11-15T03:13:48,635 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,636 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN 2024-11-15T03:13:48,636 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1beb579fcf8e161a0b9c2051c671a8f2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,636 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1beb579fcf8e161a0b9c2051c671a8f2, ASSIGN 2024-11-15T03:13:48,636 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb2ead3b463edc0f2b603da7d19f1201, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,636 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb2ead3b463edc0f2b603da7d19f1201, ASSIGN 2024-11-15T03:13:48,636 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b8f1ea215e8c2f78186b80dafed1b28, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,636 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b8f1ea215e8c2f78186b80dafed1b28, ASSIGN 2024-11-15T03:13:48,636 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d0eff1b91bc9b77579a5c44f1c985985, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,636 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d0eff1b91bc9b77579a5c44f1c985985, ASSIGN 2024-11-15T03:13:48,636 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23f804d67a47ba185ffd6f6bc7b75464, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,636 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23f804d67a47ba185ffd6f6bc7b75464, ASSIGN 2024-11-15T03:13:48,636 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1beb579fcf8e161a0b9c2051c671a8f2, server=91f82191efb8,41451,1731640420638 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,636 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1b8f1ea215e8c2f78186b80dafed1b28, server=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,636 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure d0eff1b91bc9b77579a5c44f1c985985, server=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,636 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 23f804d67a47ba185ffd6f6bc7b75464, server=91f82191efb8,41451,1731640420638 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,636 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 11 msec 2024-11-15T03:13:48,637 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=65 2024-11-15T03:13:48,638 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=91f82191efb8,37523,1731640420593, table=hbase:meta, region=1588230740 2024-11-15T03:13:48,638 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 91f82191efb8,37523,1731640420593, state=OPEN 2024-11-15T03:13:48,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37523-0x1013c7bdd830002, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:48,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41451-0x1013c7bdd830003, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:48,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42051-0x1013c7bdd830000, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:48,646 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:48,646 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:48,646 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:48,646 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN to state=OFFLINE, location=null, table=test, region=5971f307349447e7f24b0dd05b96cffc to restore RIT 2024-11-15T03:13:48,646 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN to state=OFFLINE, location=null, table=test, region=e0bce1a97a9c7f320b315db443b47c23 to restore RIT 2024-11-15T03:13:48,647 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN to state=OFFLINE, location=null, table=test, region=a73d2d096e2f14a7e9365ac762927d55 to restore RIT 2024-11-15T03:13:48,647 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1beb579fcf8e161a0b9c2051c671a8f2, ASSIGN to state=OFFLINE, location=null, table=test, region=1beb579fcf8e161a0b9c2051c671a8f2 to restore RIT 2024-11-15T03:13:48,647 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb2ead3b463edc0f2b603da7d19f1201, ASSIGN to state=OFFLINE, location=null, table=test, region=bb2ead3b463edc0f2b603da7d19f1201 to restore RIT 2024-11-15T03:13:48,647 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b8f1ea215e8c2f78186b80dafed1b28, ASSIGN to state=OFFLINE, location=null, table=test, region=1b8f1ea215e8c2f78186b80dafed1b28 to restore RIT 2024-11-15T03:13:48,647 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d0eff1b91bc9b77579a5c44f1c985985, ASSIGN to state=OFFLINE, location=null, table=test, region=d0eff1b91bc9b77579a5c44f1c985985 to restore RIT 2024-11-15T03:13:48,647 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23f804d67a47ba185ffd6f6bc7b75464, ASSIGN to state=OFFLINE, location=null, table=test, region=23f804d67a47ba185ffd6f6bc7b75464 to restore RIT 2024-11-15T03:13:48,647 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731640458647 2024-11-15T03:13:48,647 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-15T03:13:48,647 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T03:13:48,648 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-15T03:13:48,650 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-15T03:13:48,650 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=77, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure d0eff1b91bc9b77579a5c44f1c985985, server=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,650 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T03:13:48,650 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=77, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure d0eff1b91bc9b77579a5c44f1c985985, server=91f82191efb8,37523,1731640420593 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T03:13:48,654 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5965b596ce7b87fabde47569d60c3f95, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,654 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6130297c45bd23340b368de2cc708c90, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,654 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5971f307349447e7f24b0dd05b96cffc, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,654 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1beb579fcf8e161a0b9c2051c671a8f2, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,655 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0f9e1e28ca8906f0f82f959a60c55ce, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,655 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4b611da4a73d4cfc99379713486e21c, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,655 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0eff1b91bc9b77579a5c44f1c985985, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,655 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48ff514f5498660a4e8fc753340bb847, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,655 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5219cfe492c74eb8312930d328964b6, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,656 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1704275babe02e36b29cfaadf5b81af, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,656 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb2ead3b463edc0f2b603da7d19f1201, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,656 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=860df8edb65f2fdd3c8bc00f89f94ba0, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,656 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eaf85cc654192e46d3ea0904493eea12, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,656 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a73d2d096e2f14a7e9365ac762927d55, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,657 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b96380a8d7110dffa8cd7844252025dd, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,657 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fac2a1b464a5ff2d581af6baef327bde, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,657 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f804d67a47ba185ffd6f6bc7b75464, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,657 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc930ecfca79abbf57ed28c20f79230f, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,657 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ef03d91ad777a77cf66f510e28419ba, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,658 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=21df709a703798d68290a0a63322957f, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,658 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1b8f1ea215e8c2f78186b80dafed1b28, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,658 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc93af816bdf94cf920a5adc31356783, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=5 2024-11-15T03:13:48,658 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=155ef591549501ad2da24d2c559fb7f9, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,658 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6cca2d3f6a001d96f34b7a93afec8fe7, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,658 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0bce1a97a9c7f320b315db443b47c23, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,659 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17e8b3b889a606b8c6826552e784d401, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,659 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-15T03:13:48,659 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731640488659 2024-11-15T03:13:48,659 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731640548659 2024-11-15T03:13:48,659 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 11 msec 2024-11-15T03:13:48,659 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T03:13:48,659 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T03:13:48,659 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T03:13:48,659 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T03:13:48,660 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T03:13:48,660 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T03:13:48,660 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T03:13:48,660 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T03:13:48,661 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-15T03:13:48,664 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-15T03:13:48,664 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T03:13:48,664 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T03:13:48,664 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,664 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,664 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,664 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,664 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T03:13:48,665 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,665 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,665 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T03:13:48,665 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,665 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T03:13:48,665 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,665 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T03:13:48,665 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T03:13:48,665 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,665 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T03:13:48,665 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T03:13:48,666 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-15T03:13:48,666 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=dc93af816bdf94cf920a5adc31356783, REOPEN/MOVE 2024-11-15T03:13:48,669 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true 2024-11-15T03:13:48,669 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,669 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,669 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T03:13:48,669 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,669 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,669 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,669 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN 2024-11-15T03:13:48,669 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN 2024-11-15T03:13:48,669 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN 2024-11-15T03:13:48,669 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1beb579fcf8e161a0b9c2051c671a8f2, ASSIGN 2024-11-15T03:13:48,669 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb2ead3b463edc0f2b603da7d19f1201, ASSIGN 2024-11-15T03:13:48,669 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b8f1ea215e8c2f78186b80dafed1b28, ASSIGN 2024-11-15T03:13:48,669 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d0eff1b91bc9b77579a5c44f1c985985, ASSIGN 2024-11-15T03:13:48,669 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23f804d67a47ba185ffd6f6bc7b75464, ASSIGN 2024-11-15T03:13:48,669 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1beb579fcf8e161a0b9c2051c671a8f2, server=91f82191efb8,41451,1731640420638 2024-11-15T03:13:48,669 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1b8f1ea215e8c2f78186b80dafed1b28, server=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,670 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure d0eff1b91bc9b77579a5c44f1c985985, server=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,670 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,670 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,670 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,670 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,670 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,670 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,670 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,670 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,670 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN 2024-11-15T03:13:48,670 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,670 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN 2024-11-15T03:13:48,670 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,670 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN 2024-11-15T03:13:48,670 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1beb579fcf8e161a0b9c2051c671a8f2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,670 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1beb579fcf8e161a0b9c2051c671a8f2, ASSIGN 2024-11-15T03:13:48,670 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb2ead3b463edc0f2b603da7d19f1201, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,670 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb2ead3b463edc0f2b603da7d19f1201, ASSIGN 2024-11-15T03:13:48,671 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b8f1ea215e8c2f78186b80dafed1b28, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,671 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b8f1ea215e8c2f78186b80dafed1b28, ASSIGN 2024-11-15T03:13:48,671 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d0eff1b91bc9b77579a5c44f1c985985, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,671 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d0eff1b91bc9b77579a5c44f1c985985, ASSIGN 2024-11-15T03:13:48,671 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23f804d67a47ba185ffd6f6bc7b75464, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,671 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23f804d67a47ba185ffd6f6bc7b75464, ASSIGN 2024-11-15T03:13:48,671 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1beb579fcf8e161a0b9c2051c671a8f2, server=91f82191efb8,41451,1731640420638 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,671 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1b8f1ea215e8c2f78186b80dafed1b28, server=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,671 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure d0eff1b91bc9b77579a5c44f1c985985, server=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,671 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 10 msec 2024-11-15T03:13:48,672 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=65 2024-11-15T03:13:48,673 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=91f82191efb8,37523,1731640420593, table=hbase:meta, region=1588230740 2024-11-15T03:13:48,673 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 91f82191efb8,37523,1731640420593, state=OPEN 2024-11-15T03:13:48,674 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42051 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T03:13:48,674 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42051 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T03:13:48,682 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37523-0x1013c7bdd830002, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:48,682 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41451-0x1013c7bdd830003, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:48,682 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42051-0x1013c7bdd830000, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:48,683 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:48,683 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:48,683 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:48,683 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN to state=OFFLINE, location=null, table=test, region=5971f307349447e7f24b0dd05b96cffc to restore RIT 2024-11-15T03:13:48,683 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN to state=OFFLINE, location=null, table=test, region=e0bce1a97a9c7f320b315db443b47c23 to restore RIT 2024-11-15T03:13:48,683 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN to state=OFFLINE, location=null, table=test, region=a73d2d096e2f14a7e9365ac762927d55 to restore RIT 2024-11-15T03:13:48,683 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1beb579fcf8e161a0b9c2051c671a8f2, ASSIGN to state=OFFLINE, location=null, table=test, region=1beb579fcf8e161a0b9c2051c671a8f2 to restore RIT 2024-11-15T03:13:48,684 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb2ead3b463edc0f2b603da7d19f1201, ASSIGN to state=OFFLINE, location=null, table=test, region=bb2ead3b463edc0f2b603da7d19f1201 to restore RIT 2024-11-15T03:13:48,684 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b8f1ea215e8c2f78186b80dafed1b28, ASSIGN to state=OFFLINE, location=null, table=test, region=1b8f1ea215e8c2f78186b80dafed1b28 to restore RIT 2024-11-15T03:13:48,684 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d0eff1b91bc9b77579a5c44f1c985985, ASSIGN to state=OFFLINE, location=null, table=test, region=d0eff1b91bc9b77579a5c44f1c985985 to restore RIT 2024-11-15T03:13:48,684 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23f804d67a47ba185ffd6f6bc7b75464, ASSIGN to state=OFFLINE, location=null, table=test, region=23f804d67a47ba185ffd6f6bc7b75464 to restore RIT 2024-11-15T03:13:48,684 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731640458684 2024-11-15T03:13:48,684 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-15T03:13:48,684 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T03:13:48,686 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-15T03:13:48,687 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-15T03:13:48,687 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=76, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1b8f1ea215e8c2f78186b80dafed1b28, server=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,687 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T03:13:48,687 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=76, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1b8f1ea215e8c2f78186b80dafed1b28, server=91f82191efb8,37523,1731640420593 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T03:13:48,690 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5965b596ce7b87fabde47569d60c3f95, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,690 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6130297c45bd23340b368de2cc708c90, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,690 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5971f307349447e7f24b0dd05b96cffc, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,690 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1beb579fcf8e161a0b9c2051c671a8f2, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,690 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0f9e1e28ca8906f0f82f959a60c55ce, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,691 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4b611da4a73d4cfc99379713486e21c, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,691 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0eff1b91bc9b77579a5c44f1c985985, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,691 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48ff514f5498660a4e8fc753340bb847, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,691 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5219cfe492c74eb8312930d328964b6, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,691 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1704275babe02e36b29cfaadf5b81af, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,691 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb2ead3b463edc0f2b603da7d19f1201, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,691 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=860df8edb65f2fdd3c8bc00f89f94ba0, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,691 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eaf85cc654192e46d3ea0904493eea12, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,692 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a73d2d096e2f14a7e9365ac762927d55, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,692 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b96380a8d7110dffa8cd7844252025dd, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,692 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fac2a1b464a5ff2d581af6baef327bde, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,692 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f804d67a47ba185ffd6f6bc7b75464, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,692 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc930ecfca79abbf57ed28c20f79230f, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,692 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ef03d91ad777a77cf66f510e28419ba, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,692 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=21df709a703798d68290a0a63322957f, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,692 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1b8f1ea215e8c2f78186b80dafed1b28, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,693 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc93af816bdf94cf920a5adc31356783, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=5 2024-11-15T03:13:48,693 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=155ef591549501ad2da24d2c559fb7f9, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,693 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6cca2d3f6a001d96f34b7a93afec8fe7, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,693 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0bce1a97a9c7f320b315db443b47c23, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,693 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17e8b3b889a606b8c6826552e784d401, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,693 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-15T03:13:48,693 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731640488693 2024-11-15T03:13:48,693 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731640548693 2024-11-15T03:13:48,693 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-15T03:13:48,693 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T03:13:48,693 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T03:13:48,693 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T03:13:48,694 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T03:13:48,694 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T03:13:48,694 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T03:13:48,694 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T03:13:48,694 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T03:13:48,695 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-15T03:13:48,697 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-15T03:13:48,697 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T03:13:48,697 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T03:13:48,697 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,697 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,697 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,698 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,698 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T03:13:48,698 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,698 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,698 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T03:13:48,698 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,698 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,698 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T03:13:48,698 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T03:13:48,698 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,698 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T03:13:48,698 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T03:13:48,699 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-15T03:13:48,699 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=dc93af816bdf94cf920a5adc31356783, REOPEN/MOVE 2024-11-15T03:13:48,701 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true 2024-11-15T03:13:48,702 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,702 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,702 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T03:13:48,702 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,702 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,702 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,702 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN 2024-11-15T03:13:48,702 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN 2024-11-15T03:13:48,702 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN 2024-11-15T03:13:48,702 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1beb579fcf8e161a0b9c2051c671a8f2, ASSIGN 2024-11-15T03:13:48,702 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb2ead3b463edc0f2b603da7d19f1201, ASSIGN 2024-11-15T03:13:48,702 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b8f1ea215e8c2f78186b80dafed1b28, ASSIGN 2024-11-15T03:13:48,702 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d0eff1b91bc9b77579a5c44f1c985985, ASSIGN 2024-11-15T03:13:48,702 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23f804d67a47ba185ffd6f6bc7b75464, ASSIGN 2024-11-15T03:13:48,702 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1beb579fcf8e161a0b9c2051c671a8f2, server=91f82191efb8,41451,1731640420638 2024-11-15T03:13:48,702 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1b8f1ea215e8c2f78186b80dafed1b28, server=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,702 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,702 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,703 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,703 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,703 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,703 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,703 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,703 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,703 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN 2024-11-15T03:13:48,703 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,703 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN 2024-11-15T03:13:48,703 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,703 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN 2024-11-15T03:13:48,703 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1beb579fcf8e161a0b9c2051c671a8f2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,703 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1beb579fcf8e161a0b9c2051c671a8f2, ASSIGN 2024-11-15T03:13:48,703 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb2ead3b463edc0f2b603da7d19f1201, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,703 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb2ead3b463edc0f2b603da7d19f1201, ASSIGN 2024-11-15T03:13:48,703 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b8f1ea215e8c2f78186b80dafed1b28, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,703 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b8f1ea215e8c2f78186b80dafed1b28, ASSIGN 2024-11-15T03:13:48,703 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d0eff1b91bc9b77579a5c44f1c985985, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,703 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d0eff1b91bc9b77579a5c44f1c985985, ASSIGN 2024-11-15T03:13:48,704 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23f804d67a47ba185ffd6f6bc7b75464, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,704 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23f804d67a47ba185ffd6f6bc7b75464, ASSIGN 2024-11-15T03:13:48,704 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1beb579fcf8e161a0b9c2051c671a8f2, server=91f82191efb8,41451,1731640420638 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,704 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1b8f1ea215e8c2f78186b80dafed1b28, server=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,704 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-15T03:13:48,705 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=65 2024-11-15T03:13:48,705 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=91f82191efb8,37523,1731640420593, table=hbase:meta, region=1588230740 2024-11-15T03:13:48,705 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 91f82191efb8,37523,1731640420593, state=OPEN 2024-11-15T03:13:48,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37523-0x1013c7bdd830002, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:48,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41451-0x1013c7bdd830003, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:48,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42051-0x1013c7bdd830000, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:48,714 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:48,714 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:48,714 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:48,715 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN to state=OFFLINE, location=null, table=test, region=5971f307349447e7f24b0dd05b96cffc to restore RIT 2024-11-15T03:13:48,715 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN to state=OFFLINE, location=null, table=test, region=e0bce1a97a9c7f320b315db443b47c23 to restore RIT 2024-11-15T03:13:48,715 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN to state=OFFLINE, location=null, table=test, region=a73d2d096e2f14a7e9365ac762927d55 to restore RIT 2024-11-15T03:13:48,715 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1beb579fcf8e161a0b9c2051c671a8f2, ASSIGN to state=OFFLINE, location=null, table=test, region=1beb579fcf8e161a0b9c2051c671a8f2 to restore RIT 2024-11-15T03:13:48,715 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb2ead3b463edc0f2b603da7d19f1201, ASSIGN to state=OFFLINE, location=null, table=test, region=bb2ead3b463edc0f2b603da7d19f1201 to restore RIT 2024-11-15T03:13:48,715 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b8f1ea215e8c2f78186b80dafed1b28, ASSIGN to state=OFFLINE, location=null, table=test, region=1b8f1ea215e8c2f78186b80dafed1b28 to restore RIT 2024-11-15T03:13:48,715 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d0eff1b91bc9b77579a5c44f1c985985, ASSIGN to state=OFFLINE, location=null, table=test, region=d0eff1b91bc9b77579a5c44f1c985985 to restore RIT 2024-11-15T03:13:48,715 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23f804d67a47ba185ffd6f6bc7b75464, ASSIGN to state=OFFLINE, location=null, table=test, region=23f804d67a47ba185ffd6f6bc7b75464 to restore RIT 2024-11-15T03:13:48,716 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731640458716 2024-11-15T03:13:48,716 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-15T03:13:48,716 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T03:13:48,717 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-15T03:13:48,718 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-15T03:13:48,719 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1beb579fcf8e161a0b9c2051c671a8f2, server=91f82191efb8,41451,1731640420638 2024-11-15T03:13:48,719 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T03:13:48,719 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1beb579fcf8e161a0b9c2051c671a8f2, server=91f82191efb8,41451,1731640420638 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T03:13:48,721 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5965b596ce7b87fabde47569d60c3f95, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,721 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6130297c45bd23340b368de2cc708c90, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,721 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5971f307349447e7f24b0dd05b96cffc, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,722 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1beb579fcf8e161a0b9c2051c671a8f2, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,722 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0f9e1e28ca8906f0f82f959a60c55ce, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,722 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4b611da4a73d4cfc99379713486e21c, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,722 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0eff1b91bc9b77579a5c44f1c985985, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,722 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48ff514f5498660a4e8fc753340bb847, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,722 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5219cfe492c74eb8312930d328964b6, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,722 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1704275babe02e36b29cfaadf5b81af, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,722 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb2ead3b463edc0f2b603da7d19f1201, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,722 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=860df8edb65f2fdd3c8bc00f89f94ba0, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,723 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eaf85cc654192e46d3ea0904493eea12, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,723 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a73d2d096e2f14a7e9365ac762927d55, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,723 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b96380a8d7110dffa8cd7844252025dd, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,723 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fac2a1b464a5ff2d581af6baef327bde, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,723 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f804d67a47ba185ffd6f6bc7b75464, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,723 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc930ecfca79abbf57ed28c20f79230f, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,723 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ef03d91ad777a77cf66f510e28419ba, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,723 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=21df709a703798d68290a0a63322957f, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,723 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1b8f1ea215e8c2f78186b80dafed1b28, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,724 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc93af816bdf94cf920a5adc31356783, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=5 2024-11-15T03:13:48,724 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=155ef591549501ad2da24d2c559fb7f9, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,724 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6cca2d3f6a001d96f34b7a93afec8fe7, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,724 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0bce1a97a9c7f320b315db443b47c23, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,724 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17e8b3b889a606b8c6826552e784d401, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,724 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-15T03:13:48,724 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731640488724 2024-11-15T03:13:48,724 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731640548724 2024-11-15T03:13:48,724 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-15T03:13:48,724 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T03:13:48,724 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T03:13:48,724 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T03:13:48,724 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T03:13:48,725 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T03:13:48,725 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T03:13:48,725 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T03:13:48,725 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T03:13:48,726 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-15T03:13:48,727 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-15T03:13:48,727 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T03:13:48,727 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T03:13:48,727 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,728 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,728 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,728 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,728 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T03:13:48,728 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,728 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,728 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,728 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,728 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T03:13:48,728 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T03:13:48,728 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,728 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T03:13:48,728 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T03:13:48,729 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-15T03:13:48,729 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=dc93af816bdf94cf920a5adc31356783, REOPEN/MOVE 2024-11-15T03:13:48,731 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true 2024-11-15T03:13:48,731 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,731 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,731 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T03:13:48,731 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,731 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,731 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,731 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN 2024-11-15T03:13:48,731 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN 2024-11-15T03:13:48,731 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN 2024-11-15T03:13:48,731 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1beb579fcf8e161a0b9c2051c671a8f2, ASSIGN 2024-11-15T03:13:48,731 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb2ead3b463edc0f2b603da7d19f1201, ASSIGN 2024-11-15T03:13:48,731 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b8f1ea215e8c2f78186b80dafed1b28, ASSIGN 2024-11-15T03:13:48,731 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d0eff1b91bc9b77579a5c44f1c985985, ASSIGN 2024-11-15T03:13:48,731 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23f804d67a47ba185ffd6f6bc7b75464, ASSIGN 2024-11-15T03:13:48,731 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1beb579fcf8e161a0b9c2051c671a8f2, server=91f82191efb8,41451,1731640420638 2024-11-15T03:13:48,731 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,731 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,731 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,731 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,731 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,731 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,731 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,731 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,731 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN 2024-11-15T03:13:48,732 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,732 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN 2024-11-15T03:13:48,732 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,732 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN 2024-11-15T03:13:48,732 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1beb579fcf8e161a0b9c2051c671a8f2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,732 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1beb579fcf8e161a0b9c2051c671a8f2, ASSIGN 2024-11-15T03:13:48,732 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb2ead3b463edc0f2b603da7d19f1201, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,732 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb2ead3b463edc0f2b603da7d19f1201, ASSIGN 2024-11-15T03:13:48,732 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b8f1ea215e8c2f78186b80dafed1b28, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,732 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b8f1ea215e8c2f78186b80dafed1b28, ASSIGN 2024-11-15T03:13:48,732 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d0eff1b91bc9b77579a5c44f1c985985, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,732 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d0eff1b91bc9b77579a5c44f1c985985, ASSIGN 2024-11-15T03:13:48,732 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23f804d67a47ba185ffd6f6bc7b75464, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,732 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23f804d67a47ba185ffd6f6bc7b75464, ASSIGN 2024-11-15T03:13:48,732 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1beb579fcf8e161a0b9c2051c671a8f2, server=91f82191efb8,41451,1731640420638 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,732 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-15T03:13:48,733 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=65 2024-11-15T03:13:48,733 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=91f82191efb8,37523,1731640420593, table=hbase:meta, region=1588230740 2024-11-15T03:13:48,733 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 91f82191efb8,37523,1731640420593, state=OPEN 2024-11-15T03:13:48,746 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37523-0x1013c7bdd830002, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:48,746 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42051-0x1013c7bdd830000, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:48,746 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41451-0x1013c7bdd830003, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:48,746 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:48,746 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:48,746 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:48,746 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN to state=OFFLINE, location=null, table=test, region=5971f307349447e7f24b0dd05b96cffc to restore RIT 2024-11-15T03:13:48,747 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN to state=OFFLINE, location=null, table=test, region=e0bce1a97a9c7f320b315db443b47c23 to restore RIT 2024-11-15T03:13:48,747 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN to state=OFFLINE, location=null, table=test, region=a73d2d096e2f14a7e9365ac762927d55 to restore RIT 2024-11-15T03:13:48,747 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1beb579fcf8e161a0b9c2051c671a8f2, ASSIGN to state=OFFLINE, location=null, table=test, region=1beb579fcf8e161a0b9c2051c671a8f2 to restore RIT 2024-11-15T03:13:48,747 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb2ead3b463edc0f2b603da7d19f1201, ASSIGN to state=OFFLINE, location=null, table=test, region=bb2ead3b463edc0f2b603da7d19f1201 to restore RIT 2024-11-15T03:13:48,747 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b8f1ea215e8c2f78186b80dafed1b28, ASSIGN to state=OFFLINE, location=null, table=test, region=1b8f1ea215e8c2f78186b80dafed1b28 to restore RIT 2024-11-15T03:13:48,747 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d0eff1b91bc9b77579a5c44f1c985985, ASSIGN to state=OFFLINE, location=null, table=test, region=d0eff1b91bc9b77579a5c44f1c985985 to restore RIT 2024-11-15T03:13:48,747 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23f804d67a47ba185ffd6f6bc7b75464, ASSIGN to state=OFFLINE, location=null, table=test, region=23f804d67a47ba185ffd6f6bc7b75464 to restore RIT 2024-11-15T03:13:48,747 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731640458747 2024-11-15T03:13:48,747 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-15T03:13:48,747 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T03:13:48,748 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-15T03:13:48,749 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-15T03:13:48,750 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23f804d67a47ba185ffd6f6bc7b75464, ASSIGN 2024-11-15T03:13:48,750 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T03:13:48,750 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23f804d67a47ba185ffd6f6bc7b75464, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T03:13:48,754 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5965b596ce7b87fabde47569d60c3f95, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,754 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6130297c45bd23340b368de2cc708c90, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,754 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5971f307349447e7f24b0dd05b96cffc, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,754 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1beb579fcf8e161a0b9c2051c671a8f2, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,754 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0f9e1e28ca8906f0f82f959a60c55ce, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,754 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4b611da4a73d4cfc99379713486e21c, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,755 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0eff1b91bc9b77579a5c44f1c985985, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,755 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48ff514f5498660a4e8fc753340bb847, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,755 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5219cfe492c74eb8312930d328964b6, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,755 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1704275babe02e36b29cfaadf5b81af, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,756 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb2ead3b463edc0f2b603da7d19f1201, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,756 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=860df8edb65f2fdd3c8bc00f89f94ba0, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,756 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eaf85cc654192e46d3ea0904493eea12, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,756 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a73d2d096e2f14a7e9365ac762927d55, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,756 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b96380a8d7110dffa8cd7844252025dd, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,757 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fac2a1b464a5ff2d581af6baef327bde, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,757 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f804d67a47ba185ffd6f6bc7b75464, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,757 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc930ecfca79abbf57ed28c20f79230f, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,757 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ef03d91ad777a77cf66f510e28419ba, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,757 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=21df709a703798d68290a0a63322957f, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,757 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1b8f1ea215e8c2f78186b80dafed1b28, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,757 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc93af816bdf94cf920a5adc31356783, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=5 2024-11-15T03:13:48,757 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=155ef591549501ad2da24d2c559fb7f9, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,758 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6cca2d3f6a001d96f34b7a93afec8fe7, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,758 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0bce1a97a9c7f320b315db443b47c23, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,758 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17e8b3b889a606b8c6826552e784d401, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,758 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-15T03:13:48,758 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731640488758 2024-11-15T03:13:48,758 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731640548758 2024-11-15T03:13:48,758 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 10 msec 2024-11-15T03:13:48,758 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T03:13:48,758 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T03:13:48,758 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T03:13:48,758 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T03:13:48,759 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T03:13:48,759 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T03:13:48,759 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T03:13:48,759 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T03:13:48,760 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-15T03:13:48,761 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-15T03:13:48,761 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T03:13:48,761 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T03:13:48,761 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,761 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,762 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,762 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,762 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,762 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,762 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,762 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,762 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T03:13:48,762 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T03:13:48,762 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,762 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T03:13:48,762 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T03:13:48,763 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-15T03:13:48,763 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=dc93af816bdf94cf920a5adc31356783, REOPEN/MOVE 2024-11-15T03:13:48,764 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true 2024-11-15T03:13:48,764 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,764 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,764 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T03:13:48,764 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,764 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,764 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,764 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN 2024-11-15T03:13:48,764 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN 2024-11-15T03:13:48,765 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN 2024-11-15T03:13:48,765 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1beb579fcf8e161a0b9c2051c671a8f2, ASSIGN 2024-11-15T03:13:48,765 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb2ead3b463edc0f2b603da7d19f1201, ASSIGN 2024-11-15T03:13:48,765 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b8f1ea215e8c2f78186b80dafed1b28, ASSIGN 2024-11-15T03:13:48,765 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d0eff1b91bc9b77579a5c44f1c985985, ASSIGN 2024-11-15T03:13:48,765 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23f804d67a47ba185ffd6f6bc7b75464, ASSIGN 2024-11-15T03:13:48,765 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,765 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,765 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,765 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,765 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,765 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,765 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,765 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,765 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN 2024-11-15T03:13:48,765 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,765 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN 2024-11-15T03:13:48,765 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,765 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN 2024-11-15T03:13:48,765 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1beb579fcf8e161a0b9c2051c671a8f2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,765 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1beb579fcf8e161a0b9c2051c671a8f2, ASSIGN 2024-11-15T03:13:48,765 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb2ead3b463edc0f2b603da7d19f1201, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,765 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb2ead3b463edc0f2b603da7d19f1201, ASSIGN 2024-11-15T03:13:48,765 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b8f1ea215e8c2f78186b80dafed1b28, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,766 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b8f1ea215e8c2f78186b80dafed1b28, ASSIGN 2024-11-15T03:13:48,766 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d0eff1b91bc9b77579a5c44f1c985985, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,766 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d0eff1b91bc9b77579a5c44f1c985985, ASSIGN 2024-11-15T03:13:48,766 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23f804d67a47ba185ffd6f6bc7b75464, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,766 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23f804d67a47ba185ffd6f6bc7b75464, ASSIGN 2024-11-15T03:13:48,766 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-15T03:13:48,767 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=65 2024-11-15T03:13:48,767 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=91f82191efb8,37523,1731640420593, table=hbase:meta, region=1588230740 2024-11-15T03:13:48,767 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 91f82191efb8,37523,1731640420593, state=OPEN 2024-11-15T03:13:48,777 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37523-0x1013c7bdd830002, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:48,777 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41451-0x1013c7bdd830003, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:48,777 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42051-0x1013c7bdd830000, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:48,777 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:48,777 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:48,777 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:48,778 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN to state=OFFLINE, location=null, table=test, region=5971f307349447e7f24b0dd05b96cffc to restore RIT 2024-11-15T03:13:48,778 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN to state=OFFLINE, location=null, table=test, region=e0bce1a97a9c7f320b315db443b47c23 to restore RIT 2024-11-15T03:13:48,778 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN to state=OFFLINE, location=null, table=test, region=a73d2d096e2f14a7e9365ac762927d55 to restore RIT 2024-11-15T03:13:48,778 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1beb579fcf8e161a0b9c2051c671a8f2, ASSIGN to state=OFFLINE, location=null, table=test, region=1beb579fcf8e161a0b9c2051c671a8f2 to restore RIT 2024-11-15T03:13:48,778 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb2ead3b463edc0f2b603da7d19f1201, ASSIGN to state=OFFLINE, location=null, table=test, region=bb2ead3b463edc0f2b603da7d19f1201 to restore RIT 2024-11-15T03:13:48,778 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b8f1ea215e8c2f78186b80dafed1b28, ASSIGN to state=OFFLINE, location=null, table=test, region=1b8f1ea215e8c2f78186b80dafed1b28 to restore RIT 2024-11-15T03:13:48,778 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d0eff1b91bc9b77579a5c44f1c985985, ASSIGN to state=OFFLINE, location=null, table=test, region=d0eff1b91bc9b77579a5c44f1c985985 to restore RIT 2024-11-15T03:13:48,778 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=23f804d67a47ba185ffd6f6bc7b75464, ASSIGN to state=OFFLINE, location=null, table=test, region=23f804d67a47ba185ffd6f6bc7b75464 to restore RIT 2024-11-15T03:13:48,779 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731640458779 2024-11-15T03:13:48,779 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-15T03:13:48,779 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T03:13:48,780 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-15T03:13:48,781 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-15T03:13:48,781 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d0eff1b91bc9b77579a5c44f1c985985, ASSIGN 2024-11-15T03:13:48,781 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T03:13:48,781 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d0eff1b91bc9b77579a5c44f1c985985, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T03:13:48,784 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5965b596ce7b87fabde47569d60c3f95, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,785 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6130297c45bd23340b368de2cc708c90, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,785 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5971f307349447e7f24b0dd05b96cffc, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,785 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1beb579fcf8e161a0b9c2051c671a8f2, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,785 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0f9e1e28ca8906f0f82f959a60c55ce, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,785 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4b611da4a73d4cfc99379713486e21c, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,786 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0eff1b91bc9b77579a5c44f1c985985, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,786 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48ff514f5498660a4e8fc753340bb847, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,786 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5219cfe492c74eb8312930d328964b6, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,786 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1704275babe02e36b29cfaadf5b81af, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,786 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb2ead3b463edc0f2b603da7d19f1201, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,786 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=860df8edb65f2fdd3c8bc00f89f94ba0, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,786 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eaf85cc654192e46d3ea0904493eea12, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,787 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a73d2d096e2f14a7e9365ac762927d55, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,787 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b96380a8d7110dffa8cd7844252025dd, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,787 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fac2a1b464a5ff2d581af6baef327bde, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,787 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f804d67a47ba185ffd6f6bc7b75464, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,787 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc930ecfca79abbf57ed28c20f79230f, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,787 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ef03d91ad777a77cf66f510e28419ba, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,788 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=21df709a703798d68290a0a63322957f, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,788 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1b8f1ea215e8c2f78186b80dafed1b28, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,788 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc93af816bdf94cf920a5adc31356783, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=5 2024-11-15T03:13:48,788 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=155ef591549501ad2da24d2c559fb7f9, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,788 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6cca2d3f6a001d96f34b7a93afec8fe7, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,788 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0bce1a97a9c7f320b315db443b47c23, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,789 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17e8b3b889a606b8c6826552e784d401, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,789 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-15T03:13:48,789 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731640488789 2024-11-15T03:13:48,789 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731640548789 2024-11-15T03:13:48,789 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-15T03:13:48,789 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T03:13:48,789 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T03:13:48,789 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T03:13:48,789 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T03:13:48,789 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T03:13:48,790 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T03:13:48,790 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T03:13:48,790 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T03:13:48,791 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-15T03:13:48,793 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-15T03:13:48,793 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T03:13:48,793 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T03:13:48,793 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,793 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,793 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,793 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,793 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,793 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,794 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,794 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T03:13:48,794 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T03:13:48,794 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,794 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T03:13:48,794 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T03:13:48,795 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-15T03:13:48,795 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=dc93af816bdf94cf920a5adc31356783, REOPEN/MOVE 2024-11-15T03:13:48,796 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true 2024-11-15T03:13:48,796 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,796 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,796 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T03:13:48,796 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,796 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,796 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,796 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN 2024-11-15T03:13:48,796 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN 2024-11-15T03:13:48,796 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN 2024-11-15T03:13:48,796 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1beb579fcf8e161a0b9c2051c671a8f2, ASSIGN 2024-11-15T03:13:48,796 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb2ead3b463edc0f2b603da7d19f1201, ASSIGN 2024-11-15T03:13:48,796 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b8f1ea215e8c2f78186b80dafed1b28, ASSIGN 2024-11-15T03:13:48,796 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d0eff1b91bc9b77579a5c44f1c985985, ASSIGN 2024-11-15T03:13:48,796 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,797 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,797 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,797 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,797 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,797 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,797 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,797 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,797 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN 2024-11-15T03:13:48,797 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,797 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN 2024-11-15T03:13:48,797 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,797 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN 2024-11-15T03:13:48,797 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1beb579fcf8e161a0b9c2051c671a8f2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,797 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1beb579fcf8e161a0b9c2051c671a8f2, ASSIGN 2024-11-15T03:13:48,797 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb2ead3b463edc0f2b603da7d19f1201, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,797 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb2ead3b463edc0f2b603da7d19f1201, ASSIGN 2024-11-15T03:13:48,797 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b8f1ea215e8c2f78186b80dafed1b28, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,798 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b8f1ea215e8c2f78186b80dafed1b28, ASSIGN 2024-11-15T03:13:48,798 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d0eff1b91bc9b77579a5c44f1c985985, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,798 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d0eff1b91bc9b77579a5c44f1c985985, ASSIGN 2024-11-15T03:13:48,798 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-15T03:13:48,799 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=65 2024-11-15T03:13:48,799 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=91f82191efb8,37523,1731640420593, table=hbase:meta, region=1588230740 2024-11-15T03:13:48,799 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 91f82191efb8,37523,1731640420593, state=OPEN 2024-11-15T03:13:48,809 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42051-0x1013c7bdd830000, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:48,809 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37523-0x1013c7bdd830002, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:48,809 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41451-0x1013c7bdd830003, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:48,809 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:48,809 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:48,809 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:48,810 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN to state=OFFLINE, location=null, table=test, region=5971f307349447e7f24b0dd05b96cffc to restore RIT 2024-11-15T03:13:48,810 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN to state=OFFLINE, location=null, table=test, region=e0bce1a97a9c7f320b315db443b47c23 to restore RIT 2024-11-15T03:13:48,810 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN to state=OFFLINE, location=null, table=test, region=a73d2d096e2f14a7e9365ac762927d55 to restore RIT 2024-11-15T03:13:48,810 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1beb579fcf8e161a0b9c2051c671a8f2, ASSIGN to state=OFFLINE, location=null, table=test, region=1beb579fcf8e161a0b9c2051c671a8f2 to restore RIT 2024-11-15T03:13:48,810 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb2ead3b463edc0f2b603da7d19f1201, ASSIGN to state=OFFLINE, location=null, table=test, region=bb2ead3b463edc0f2b603da7d19f1201 to restore RIT 2024-11-15T03:13:48,810 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b8f1ea215e8c2f78186b80dafed1b28, ASSIGN to state=OFFLINE, location=null, table=test, region=1b8f1ea215e8c2f78186b80dafed1b28 to restore RIT 2024-11-15T03:13:48,810 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d0eff1b91bc9b77579a5c44f1c985985, ASSIGN to state=OFFLINE, location=null, table=test, region=d0eff1b91bc9b77579a5c44f1c985985 to restore RIT 2024-11-15T03:13:48,810 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731640458810 2024-11-15T03:13:48,810 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-15T03:13:48,810 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T03:13:48,812 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-15T03:13:48,813 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-15T03:13:48,813 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b8f1ea215e8c2f78186b80dafed1b28, ASSIGN 2024-11-15T03:13:48,814 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T03:13:48,814 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b8f1ea215e8c2f78186b80dafed1b28, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T03:13:48,816 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5965b596ce7b87fabde47569d60c3f95, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,816 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6130297c45bd23340b368de2cc708c90, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,816 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5971f307349447e7f24b0dd05b96cffc, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,817 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1beb579fcf8e161a0b9c2051c671a8f2, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,817 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0f9e1e28ca8906f0f82f959a60c55ce, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,817 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4b611da4a73d4cfc99379713486e21c, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,817 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0eff1b91bc9b77579a5c44f1c985985, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,817 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48ff514f5498660a4e8fc753340bb847, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,817 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5219cfe492c74eb8312930d328964b6, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,817 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1704275babe02e36b29cfaadf5b81af, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,818 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb2ead3b463edc0f2b603da7d19f1201, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,818 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=860df8edb65f2fdd3c8bc00f89f94ba0, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,818 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eaf85cc654192e46d3ea0904493eea12, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,818 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a73d2d096e2f14a7e9365ac762927d55, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,818 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b96380a8d7110dffa8cd7844252025dd, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,818 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fac2a1b464a5ff2d581af6baef327bde, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,819 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f804d67a47ba185ffd6f6bc7b75464, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,819 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc930ecfca79abbf57ed28c20f79230f, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,819 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ef03d91ad777a77cf66f510e28419ba, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,819 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=21df709a703798d68290a0a63322957f, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,819 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1b8f1ea215e8c2f78186b80dafed1b28, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,819 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc93af816bdf94cf920a5adc31356783, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=5 2024-11-15T03:13:48,820 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=155ef591549501ad2da24d2c559fb7f9, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,820 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6cca2d3f6a001d96f34b7a93afec8fe7, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,820 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0bce1a97a9c7f320b315db443b47c23, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,820 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17e8b3b889a606b8c6826552e784d401, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,820 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-15T03:13:48,820 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731640488820 2024-11-15T03:13:48,820 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731640548820 2024-11-15T03:13:48,820 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-15T03:13:48,820 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T03:13:48,820 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T03:13:48,820 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T03:13:48,821 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T03:13:48,821 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T03:13:48,821 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T03:13:48,821 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T03:13:48,821 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T03:13:48,822 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-15T03:13:48,825 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-15T03:13:48,825 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T03:13:48,825 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T03:13:48,825 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,825 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,825 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,825 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,825 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,825 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,825 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T03:13:48,825 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T03:13:48,825 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,825 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T03:13:48,825 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T03:13:48,826 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-15T03:13:48,826 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=dc93af816bdf94cf920a5adc31356783, REOPEN/MOVE 2024-11-15T03:13:48,828 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true 2024-11-15T03:13:48,828 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,828 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,828 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T03:13:48,828 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,828 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,828 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,828 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN 2024-11-15T03:13:48,828 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN 2024-11-15T03:13:48,828 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN 2024-11-15T03:13:48,828 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1beb579fcf8e161a0b9c2051c671a8f2, ASSIGN 2024-11-15T03:13:48,828 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb2ead3b463edc0f2b603da7d19f1201, ASSIGN 2024-11-15T03:13:48,828 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b8f1ea215e8c2f78186b80dafed1b28, ASSIGN 2024-11-15T03:13:48,828 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,829 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,829 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,829 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,829 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,829 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,829 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,829 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,829 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN 2024-11-15T03:13:48,829 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,829 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN 2024-11-15T03:13:48,829 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,829 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN 2024-11-15T03:13:48,829 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1beb579fcf8e161a0b9c2051c671a8f2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,829 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1beb579fcf8e161a0b9c2051c671a8f2, ASSIGN 2024-11-15T03:13:48,829 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb2ead3b463edc0f2b603da7d19f1201, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,829 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb2ead3b463edc0f2b603da7d19f1201, ASSIGN 2024-11-15T03:13:48,829 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b8f1ea215e8c2f78186b80dafed1b28, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,829 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b8f1ea215e8c2f78186b80dafed1b28, ASSIGN 2024-11-15T03:13:48,829 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-15T03:13:48,831 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=65 2024-11-15T03:13:48,831 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=91f82191efb8,37523,1731640420593, table=hbase:meta, region=1588230740 2024-11-15T03:13:48,831 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 91f82191efb8,37523,1731640420593, state=OPEN 2024-11-15T03:13:48,840 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42051-0x1013c7bdd830000, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:48,840 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41451-0x1013c7bdd830003, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:48,840 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37523-0x1013c7bdd830002, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:48,841 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:48,841 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:48,841 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:48,841 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN to state=OFFLINE, location=null, table=test, region=5971f307349447e7f24b0dd05b96cffc to restore RIT 2024-11-15T03:13:48,841 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN to state=OFFLINE, location=null, table=test, region=e0bce1a97a9c7f320b315db443b47c23 to restore RIT 2024-11-15T03:13:48,841 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN to state=OFFLINE, location=null, table=test, region=a73d2d096e2f14a7e9365ac762927d55 to restore RIT 2024-11-15T03:13:48,841 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1beb579fcf8e161a0b9c2051c671a8f2, ASSIGN to state=OFFLINE, location=null, table=test, region=1beb579fcf8e161a0b9c2051c671a8f2 to restore RIT 2024-11-15T03:13:48,842 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb2ead3b463edc0f2b603da7d19f1201, ASSIGN to state=OFFLINE, location=null, table=test, region=bb2ead3b463edc0f2b603da7d19f1201 to restore RIT 2024-11-15T03:13:48,842 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b8f1ea215e8c2f78186b80dafed1b28, ASSIGN to state=OFFLINE, location=null, table=test, region=1b8f1ea215e8c2f78186b80dafed1b28 to restore RIT 2024-11-15T03:13:48,842 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731640458842 2024-11-15T03:13:48,842 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-15T03:13:48,842 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T03:13:48,843 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-15T03:13:48,844 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-15T03:13:48,844 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb2ead3b463edc0f2b603da7d19f1201, ASSIGN 2024-11-15T03:13:48,844 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T03:13:48,844 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb2ead3b463edc0f2b603da7d19f1201, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T03:13:48,848 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5965b596ce7b87fabde47569d60c3f95, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,849 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6130297c45bd23340b368de2cc708c90, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,849 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5971f307349447e7f24b0dd05b96cffc, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,849 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1beb579fcf8e161a0b9c2051c671a8f2, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,849 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0f9e1e28ca8906f0f82f959a60c55ce, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,849 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4b611da4a73d4cfc99379713486e21c, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,849 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0eff1b91bc9b77579a5c44f1c985985, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,849 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48ff514f5498660a4e8fc753340bb847, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,849 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5219cfe492c74eb8312930d328964b6, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,850 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1704275babe02e36b29cfaadf5b81af, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,850 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb2ead3b463edc0f2b603da7d19f1201, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,850 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=860df8edb65f2fdd3c8bc00f89f94ba0, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,850 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eaf85cc654192e46d3ea0904493eea12, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,850 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a73d2d096e2f14a7e9365ac762927d55, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,850 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b96380a8d7110dffa8cd7844252025dd, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,850 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fac2a1b464a5ff2d581af6baef327bde, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,851 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f804d67a47ba185ffd6f6bc7b75464, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,851 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc930ecfca79abbf57ed28c20f79230f, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,851 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ef03d91ad777a77cf66f510e28419ba, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,851 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=21df709a703798d68290a0a63322957f, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,851 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1b8f1ea215e8c2f78186b80dafed1b28, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,851 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc93af816bdf94cf920a5adc31356783, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=5 2024-11-15T03:13:48,851 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=155ef591549501ad2da24d2c559fb7f9, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,852 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6cca2d3f6a001d96f34b7a93afec8fe7, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,852 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0bce1a97a9c7f320b315db443b47c23, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,852 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17e8b3b889a606b8c6826552e784d401, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,852 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-15T03:13:48,852 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731640488852 2024-11-15T03:13:48,852 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731640548852 2024-11-15T03:13:48,852 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 10 msec 2024-11-15T03:13:48,852 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T03:13:48,852 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T03:13:48,853 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T03:13:48,853 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T03:13:48,853 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T03:13:48,853 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T03:13:48,853 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T03:13:48,853 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T03:13:48,854 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-15T03:13:48,857 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-15T03:13:48,857 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T03:13:48,857 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T03:13:48,857 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,857 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,857 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,857 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,857 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,857 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T03:13:48,857 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T03:13:48,857 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,857 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T03:13:48,857 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T03:13:48,858 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-15T03:13:48,858 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=dc93af816bdf94cf920a5adc31356783, REOPEN/MOVE 2024-11-15T03:13:48,859 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true 2024-11-15T03:13:48,860 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,860 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,860 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T03:13:48,860 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,860 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,860 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,860 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN 2024-11-15T03:13:48,860 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN 2024-11-15T03:13:48,860 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN 2024-11-15T03:13:48,860 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1beb579fcf8e161a0b9c2051c671a8f2, ASSIGN 2024-11-15T03:13:48,860 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb2ead3b463edc0f2b603da7d19f1201, ASSIGN 2024-11-15T03:13:48,860 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,860 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,860 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,860 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,860 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,860 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,860 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,860 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,860 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN 2024-11-15T03:13:48,860 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,861 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN 2024-11-15T03:13:48,861 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,861 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN 2024-11-15T03:13:48,861 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1beb579fcf8e161a0b9c2051c671a8f2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,861 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1beb579fcf8e161a0b9c2051c671a8f2, ASSIGN 2024-11-15T03:13:48,861 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb2ead3b463edc0f2b603da7d19f1201, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,861 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb2ead3b463edc0f2b603da7d19f1201, ASSIGN 2024-11-15T03:13:48,861 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-15T03:13:48,862 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=65 2024-11-15T03:13:48,862 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=91f82191efb8,37523,1731640420593, table=hbase:meta, region=1588230740 2024-11-15T03:13:48,862 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 91f82191efb8,37523,1731640420593, state=OPEN 2024-11-15T03:13:48,872 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41451-0x1013c7bdd830003, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:48,872 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42051-0x1013c7bdd830000, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:48,872 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37523-0x1013c7bdd830002, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:48,872 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:48,872 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:48,872 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:48,873 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN to state=OFFLINE, location=null, table=test, region=5971f307349447e7f24b0dd05b96cffc to restore RIT 2024-11-15T03:13:48,873 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN to state=OFFLINE, location=null, table=test, region=e0bce1a97a9c7f320b315db443b47c23 to restore RIT 2024-11-15T03:13:48,873 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN to state=OFFLINE, location=null, table=test, region=a73d2d096e2f14a7e9365ac762927d55 to restore RIT 2024-11-15T03:13:48,873 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1beb579fcf8e161a0b9c2051c671a8f2, ASSIGN to state=OFFLINE, location=null, table=test, region=1beb579fcf8e161a0b9c2051c671a8f2 to restore RIT 2024-11-15T03:13:48,873 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bb2ead3b463edc0f2b603da7d19f1201, ASSIGN to state=OFFLINE, location=null, table=test, region=bb2ead3b463edc0f2b603da7d19f1201 to restore RIT 2024-11-15T03:13:48,874 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731640458874 2024-11-15T03:13:48,874 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-15T03:13:48,874 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T03:13:48,875 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-15T03:13:48,876 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-15T03:13:48,876 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1beb579fcf8e161a0b9c2051c671a8f2, ASSIGN 2024-11-15T03:13:48,876 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T03:13:48,876 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1beb579fcf8e161a0b9c2051c671a8f2, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T03:13:48,879 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5965b596ce7b87fabde47569d60c3f95, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,879 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6130297c45bd23340b368de2cc708c90, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,879 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5971f307349447e7f24b0dd05b96cffc, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,879 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1beb579fcf8e161a0b9c2051c671a8f2, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,880 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0f9e1e28ca8906f0f82f959a60c55ce, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,880 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4b611da4a73d4cfc99379713486e21c, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,880 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0eff1b91bc9b77579a5c44f1c985985, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,880 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48ff514f5498660a4e8fc753340bb847, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,880 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5219cfe492c74eb8312930d328964b6, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,880 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1704275babe02e36b29cfaadf5b81af, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,881 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb2ead3b463edc0f2b603da7d19f1201, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,881 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=860df8edb65f2fdd3c8bc00f89f94ba0, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,881 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eaf85cc654192e46d3ea0904493eea12, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,881 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a73d2d096e2f14a7e9365ac762927d55, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,881 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b96380a8d7110dffa8cd7844252025dd, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,881 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fac2a1b464a5ff2d581af6baef327bde, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,881 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f804d67a47ba185ffd6f6bc7b75464, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,881 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc930ecfca79abbf57ed28c20f79230f, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,882 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ef03d91ad777a77cf66f510e28419ba, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,882 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=21df709a703798d68290a0a63322957f, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,882 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1b8f1ea215e8c2f78186b80dafed1b28, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,882 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc93af816bdf94cf920a5adc31356783, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=5 2024-11-15T03:13:48,882 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=155ef591549501ad2da24d2c559fb7f9, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,882 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6cca2d3f6a001d96f34b7a93afec8fe7, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,882 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0bce1a97a9c7f320b315db443b47c23, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,883 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17e8b3b889a606b8c6826552e784d401, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,883 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-15T03:13:48,883 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731640488883 2024-11-15T03:13:48,883 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731640548883 2024-11-15T03:13:48,883 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-15T03:13:48,883 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T03:13:48,883 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T03:13:48,883 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T03:13:48,883 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T03:13:48,883 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T03:13:48,884 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T03:13:48,884 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T03:13:48,884 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T03:13:48,885 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-15T03:13:48,887 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-15T03:13:48,887 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T03:13:48,887 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T03:13:48,887 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,887 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,887 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,887 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,887 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42051 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T03:13:48,887 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T03:13:48,887 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T03:13:48,887 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,887 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T03:13:48,887 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T03:13:48,887 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42051 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T03:13:48,888 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-15T03:13:48,888 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=dc93af816bdf94cf920a5adc31356783, REOPEN/MOVE 2024-11-15T03:13:48,889 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true 2024-11-15T03:13:48,889 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,889 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,889 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T03:13:48,889 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,889 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,889 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,889 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN 2024-11-15T03:13:48,889 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN 2024-11-15T03:13:48,889 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN 2024-11-15T03:13:48,889 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1beb579fcf8e161a0b9c2051c671a8f2, ASSIGN 2024-11-15T03:13:48,889 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,889 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,889 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,889 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,890 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,890 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,890 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,890 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,890 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN 2024-11-15T03:13:48,890 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,890 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN 2024-11-15T03:13:48,890 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,890 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN 2024-11-15T03:13:48,890 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1beb579fcf8e161a0b9c2051c671a8f2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,890 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1beb579fcf8e161a0b9c2051c671a8f2, ASSIGN 2024-11-15T03:13:48,890 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-15T03:13:48,891 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=65 2024-11-15T03:13:48,891 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=91f82191efb8,37523,1731640420593, table=hbase:meta, region=1588230740 2024-11-15T03:13:48,891 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 91f82191efb8,37523,1731640420593, state=OPEN 2024-11-15T03:13:48,904 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41451-0x1013c7bdd830003, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:48,904 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42051-0x1013c7bdd830000, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:48,904 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37523-0x1013c7bdd830002, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:48,904 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:48,904 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:48,904 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:48,904 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN to state=OFFLINE, location=null, table=test, region=5971f307349447e7f24b0dd05b96cffc to restore RIT 2024-11-15T03:13:48,904 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN to state=OFFLINE, location=null, table=test, region=e0bce1a97a9c7f320b315db443b47c23 to restore RIT 2024-11-15T03:13:48,904 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN to state=OFFLINE, location=null, table=test, region=a73d2d096e2f14a7e9365ac762927d55 to restore RIT 2024-11-15T03:13:48,904 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1beb579fcf8e161a0b9c2051c671a8f2, ASSIGN to state=OFFLINE, location=null, table=test, region=1beb579fcf8e161a0b9c2051c671a8f2 to restore RIT 2024-11-15T03:13:48,905 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731640458905 2024-11-15T03:13:48,905 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-15T03:13:48,905 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T03:13:48,906 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-15T03:13:48,907 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-15T03:13:48,907 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN 2024-11-15T03:13:48,907 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T03:13:48,907 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T03:13:48,911 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5965b596ce7b87fabde47569d60c3f95, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,911 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6130297c45bd23340b368de2cc708c90, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,911 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5971f307349447e7f24b0dd05b96cffc, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,911 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1beb579fcf8e161a0b9c2051c671a8f2, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,911 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0f9e1e28ca8906f0f82f959a60c55ce, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,911 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4b611da4a73d4cfc99379713486e21c, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,912 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0eff1b91bc9b77579a5c44f1c985985, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,912 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48ff514f5498660a4e8fc753340bb847, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,912 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5219cfe492c74eb8312930d328964b6, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,912 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1704275babe02e36b29cfaadf5b81af, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,912 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb2ead3b463edc0f2b603da7d19f1201, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,912 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=860df8edb65f2fdd3c8bc00f89f94ba0, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,912 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eaf85cc654192e46d3ea0904493eea12, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,913 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a73d2d096e2f14a7e9365ac762927d55, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,913 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b96380a8d7110dffa8cd7844252025dd, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,913 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fac2a1b464a5ff2d581af6baef327bde, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,913 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f804d67a47ba185ffd6f6bc7b75464, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,913 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc930ecfca79abbf57ed28c20f79230f, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,913 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ef03d91ad777a77cf66f510e28419ba, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,913 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=21df709a703798d68290a0a63322957f, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,913 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1b8f1ea215e8c2f78186b80dafed1b28, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,913 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc93af816bdf94cf920a5adc31356783, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=5 2024-11-15T03:13:48,914 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=155ef591549501ad2da24d2c559fb7f9, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,914 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6cca2d3f6a001d96f34b7a93afec8fe7, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,914 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0bce1a97a9c7f320b315db443b47c23, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,914 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17e8b3b889a606b8c6826552e784d401, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,914 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-15T03:13:48,914 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731640488914 2024-11-15T03:13:48,914 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731640548914 2024-11-15T03:13:48,914 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-15T03:13:48,914 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T03:13:48,914 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T03:13:48,914 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T03:13:48,914 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T03:13:48,915 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T03:13:48,915 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T03:13:48,915 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T03:13:48,915 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T03:13:48,916 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-15T03:13:48,918 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-15T03:13:48,918 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T03:13:48,918 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T03:13:48,918 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,919 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,919 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,919 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T03:13:48,919 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T03:13:48,919 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,919 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T03:13:48,919 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T03:13:48,919 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-15T03:13:48,920 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=dc93af816bdf94cf920a5adc31356783, REOPEN/MOVE 2024-11-15T03:13:48,921 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true 2024-11-15T03:13:48,921 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,921 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,921 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T03:13:48,921 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,921 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,921 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,921 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN 2024-11-15T03:13:48,921 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN 2024-11-15T03:13:48,921 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN 2024-11-15T03:13:48,921 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,921 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,921 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,921 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,921 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,921 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,922 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,922 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,922 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN 2024-11-15T03:13:48,922 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,922 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN 2024-11-15T03:13:48,922 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,922 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN 2024-11-15T03:13:48,922 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-15T03:13:48,923 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=65 2024-11-15T03:13:48,923 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=91f82191efb8,37523,1731640420593, table=hbase:meta, region=1588230740 2024-11-15T03:13:48,923 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 91f82191efb8,37523,1731640420593, state=OPEN 2024-11-15T03:13:48,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37523-0x1013c7bdd830002, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:48,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41451-0x1013c7bdd830003, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:48,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42051-0x1013c7bdd830000, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:48,935 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:48,935 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:48,935 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:48,936 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN to state=OFFLINE, location=null, table=test, region=5971f307349447e7f24b0dd05b96cffc to restore RIT 2024-11-15T03:13:48,936 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN to state=OFFLINE, location=null, table=test, region=e0bce1a97a9c7f320b315db443b47c23 to restore RIT 2024-11-15T03:13:48,936 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a73d2d096e2f14a7e9365ac762927d55, ASSIGN to state=OFFLINE, location=null, table=test, region=a73d2d096e2f14a7e9365ac762927d55 to restore RIT 2024-11-15T03:13:48,936 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731640458936 2024-11-15T03:13:48,936 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-15T03:13:48,936 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T03:13:48,937 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-15T03:13:48,939 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-15T03:13:48,939 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN 2024-11-15T03:13:48,939 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T03:13:48,939 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T03:13:48,942 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5965b596ce7b87fabde47569d60c3f95, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,942 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6130297c45bd23340b368de2cc708c90, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,942 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5971f307349447e7f24b0dd05b96cffc, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,942 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1beb579fcf8e161a0b9c2051c671a8f2, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,942 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0f9e1e28ca8906f0f82f959a60c55ce, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,942 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4b611da4a73d4cfc99379713486e21c, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,942 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0eff1b91bc9b77579a5c44f1c985985, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,943 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48ff514f5498660a4e8fc753340bb847, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,943 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5219cfe492c74eb8312930d328964b6, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,943 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1704275babe02e36b29cfaadf5b81af, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,943 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb2ead3b463edc0f2b603da7d19f1201, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,943 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=860df8edb65f2fdd3c8bc00f89f94ba0, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,943 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eaf85cc654192e46d3ea0904493eea12, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,943 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a73d2d096e2f14a7e9365ac762927d55, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,944 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b96380a8d7110dffa8cd7844252025dd, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,944 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fac2a1b464a5ff2d581af6baef327bde, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,944 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f804d67a47ba185ffd6f6bc7b75464, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,944 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc930ecfca79abbf57ed28c20f79230f, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,944 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ef03d91ad777a77cf66f510e28419ba, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,944 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=21df709a703798d68290a0a63322957f, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,944 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1b8f1ea215e8c2f78186b80dafed1b28, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,945 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc93af816bdf94cf920a5adc31356783, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=5 2024-11-15T03:13:48,945 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=155ef591549501ad2da24d2c559fb7f9, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,945 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6cca2d3f6a001d96f34b7a93afec8fe7, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,945 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0bce1a97a9c7f320b315db443b47c23, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:48,945 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17e8b3b889a606b8c6826552e784d401, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:48,945 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-15T03:13:48,945 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731640488945 2024-11-15T03:13:48,945 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731640548945 2024-11-15T03:13:48,945 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-15T03:13:48,945 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T03:13:48,945 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T03:13:48,945 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T03:13:48,945 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T03:13:48,946 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T03:13:48,946 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T03:13:48,946 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T03:13:48,946 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T03:13:48,947 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-15T03:13:48,949 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-15T03:13:48,949 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T03:13:48,949 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T03:13:48,949 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,949 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,949 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T03:13:48,949 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T03:13:48,949 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:48,949 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T03:13:48,949 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T03:13:48,950 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-15T03:13:48,950 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=dc93af816bdf94cf920a5adc31356783, REOPEN/MOVE 2024-11-15T03:13:48,951 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true 2024-11-15T03:13:48,951 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,951 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,951 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T03:13:48,951 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,951 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,951 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:48,952 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN 2024-11-15T03:13:48,952 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN 2024-11-15T03:13:48,952 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,952 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,952 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,952 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,952 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,952 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,952 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:48,952 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,952 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN 2024-11-15T03:13:48,952 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:48,952 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN 2024-11-15T03:13:48,952 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-15T03:13:48,953 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=65 2024-11-15T03:13:48,953 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=91f82191efb8,37523,1731640420593, table=hbase:meta, region=1588230740 2024-11-15T03:13:48,953 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 91f82191efb8,37523,1731640420593, state=OPEN 2024-11-15T03:13:48,990 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42051 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T03:13:48,990 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42051 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T03:13:48,994 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37523-0x1013c7bdd830002, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:48,994 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42051-0x1013c7bdd830000, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:48,994 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41451-0x1013c7bdd830003, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:48,994 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:48,994 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:48,994 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:48,995 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN to state=OFFLINE, location=null, table=test, region=5971f307349447e7f24b0dd05b96cffc to restore RIT 2024-11-15T03:13:48,995 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e0bce1a97a9c7f320b315db443b47c23, ASSIGN to state=OFFLINE, location=null, table=test, region=e0bce1a97a9c7f320b315db443b47c23 to restore RIT 2024-11-15T03:13:48,995 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731640458995 2024-11-15T03:13:48,995 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-15T03:13:48,996 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T03:13:48,996 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-15T03:13:48,997 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-15T03:13:48,997 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN 2024-11-15T03:13:48,997 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T03:13:48,998 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T03:13:49,000 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5965b596ce7b87fabde47569d60c3f95, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,000 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6130297c45bd23340b368de2cc708c90, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,001 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5971f307349447e7f24b0dd05b96cffc, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,001 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1beb579fcf8e161a0b9c2051c671a8f2, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,001 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0f9e1e28ca8906f0f82f959a60c55ce, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,001 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4b611da4a73d4cfc99379713486e21c, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,001 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0eff1b91bc9b77579a5c44f1c985985, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,001 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48ff514f5498660a4e8fc753340bb847, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,001 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5219cfe492c74eb8312930d328964b6, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,001 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1704275babe02e36b29cfaadf5b81af, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,001 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb2ead3b463edc0f2b603da7d19f1201, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,001 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=860df8edb65f2fdd3c8bc00f89f94ba0, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,001 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eaf85cc654192e46d3ea0904493eea12, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,002 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a73d2d096e2f14a7e9365ac762927d55, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,002 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b96380a8d7110dffa8cd7844252025dd, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,002 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fac2a1b464a5ff2d581af6baef327bde, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,002 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f804d67a47ba185ffd6f6bc7b75464, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,002 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc930ecfca79abbf57ed28c20f79230f, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,002 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ef03d91ad777a77cf66f510e28419ba, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,002 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=21df709a703798d68290a0a63322957f, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,002 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1b8f1ea215e8c2f78186b80dafed1b28, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,002 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc93af816bdf94cf920a5adc31356783, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=5 2024-11-15T03:13:49,002 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=155ef591549501ad2da24d2c559fb7f9, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,002 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6cca2d3f6a001d96f34b7a93afec8fe7, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,002 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0bce1a97a9c7f320b315db443b47c23, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,003 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17e8b3b889a606b8c6826552e784d401, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,003 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-15T03:13:49,003 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731640489003 2024-11-15T03:13:49,003 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731640549003 2024-11-15T03:13:49,003 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 7 msec 2024-11-15T03:13:49,003 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T03:13:49,003 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T03:13:49,003 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T03:13:49,003 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T03:13:49,003 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T03:13:49,003 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T03:13:49,003 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T03:13:49,003 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T03:13:49,004 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-15T03:13:49,005 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-15T03:13:49,005 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T03:13:49,005 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T03:13:49,005 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:49,005 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T03:13:49,006 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T03:13:49,006 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:49,006 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T03:13:49,006 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T03:13:49,007 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-15T03:13:49,007 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=dc93af816bdf94cf920a5adc31356783, REOPEN/MOVE 2024-11-15T03:13:49,008 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true 2024-11-15T03:13:49,008 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:49,008 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:49,008 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T03:13:49,008 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=91f82191efb8,37523,1731640420593 2024-11-15T03:13:49,008 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:49,008 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:49,008 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN 2024-11-15T03:13:49,008 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:49,008 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:49,008 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:49,008 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:49,008 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:49,008 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:49,009 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:49,009 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:49,009 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN 2024-11-15T03:13:49,009 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-15T03:13:49,009 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=65 2024-11-15T03:13:49,009 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=91f82191efb8,37523,1731640420593, table=hbase:meta, region=1588230740 2024-11-15T03:13:49,010 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 91f82191efb8,37523,1731640420593, state=OPEN 2024-11-15T03:13:49,030 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37523-0x1013c7bdd830002, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:49,030 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42051-0x1013c7bdd830000, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:49,030 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41451-0x1013c7bdd830003, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:49,030 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:49,030 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:49,030 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:49,031 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5971f307349447e7f24b0dd05b96cffc, ASSIGN to state=OFFLINE, location=null, table=test, region=5971f307349447e7f24b0dd05b96cffc to restore RIT 2024-11-15T03:13:49,031 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731640459031 2024-11-15T03:13:49,031 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-15T03:13:49,031 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T03:13:49,032 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-15T03:13:49,033 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-15T03:13:49,033 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:49,033 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T03:13:49,033 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T03:13:49,036 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5965b596ce7b87fabde47569d60c3f95, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,036 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6130297c45bd23340b368de2cc708c90, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,036 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5971f307349447e7f24b0dd05b96cffc, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,036 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1beb579fcf8e161a0b9c2051c671a8f2, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,036 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0f9e1e28ca8906f0f82f959a60c55ce, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,036 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4b611da4a73d4cfc99379713486e21c, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,036 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0eff1b91bc9b77579a5c44f1c985985, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,036 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48ff514f5498660a4e8fc753340bb847, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,036 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5219cfe492c74eb8312930d328964b6, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,037 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1704275babe02e36b29cfaadf5b81af, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,037 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb2ead3b463edc0f2b603da7d19f1201, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,037 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=860df8edb65f2fdd3c8bc00f89f94ba0, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,037 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eaf85cc654192e46d3ea0904493eea12, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,037 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a73d2d096e2f14a7e9365ac762927d55, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,037 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b96380a8d7110dffa8cd7844252025dd, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,037 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fac2a1b464a5ff2d581af6baef327bde, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,037 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f804d67a47ba185ffd6f6bc7b75464, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,037 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc930ecfca79abbf57ed28c20f79230f, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,037 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ef03d91ad777a77cf66f510e28419ba, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,038 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=21df709a703798d68290a0a63322957f, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,038 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1b8f1ea215e8c2f78186b80dafed1b28, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,038 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc93af816bdf94cf920a5adc31356783, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=5 2024-11-15T03:13:49,038 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=155ef591549501ad2da24d2c559fb7f9, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,038 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6cca2d3f6a001d96f34b7a93afec8fe7, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,038 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0bce1a97a9c7f320b315db443b47c23, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,038 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17e8b3b889a606b8c6826552e784d401, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,038 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-15T03:13:49,038 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731640489038 2024-11-15T03:13:49,038 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731640549038 2024-11-15T03:13:49,038 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 7 msec 2024-11-15T03:13:49,039 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T03:13:49,039 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T03:13:49,039 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T03:13:49,039 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T03:13:49,039 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T03:13:49,039 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T03:13:49,039 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T03:13:49,039 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T03:13:49,040 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-15T03:13:49,041 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-15T03:13:49,041 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T03:13:49,041 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T03:13:49,041 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T03:13:49,041 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T03:13:49,041 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:49,041 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T03:13:49,042 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T03:13:49,042 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-15T03:13:49,042 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=dc93af816bdf94cf920a5adc31356783, REOPEN/MOVE 2024-11-15T03:13:49,043 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true 2024-11-15T03:13:49,043 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:49,043 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:49,043 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T03:13:49,043 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=91f82191efb8,37523,1731640420593 2024-11-15T03:13:49,043 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:49,043 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:49,043 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:49,043 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:49,043 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:49,043 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:49,043 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:49,043 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:49,043 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:49,043 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-15T03:13:49,044 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=65 2024-11-15T03:13:49,044 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=91f82191efb8,37523,1731640420593, table=hbase:meta, region=1588230740 2024-11-15T03:13:49,044 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 91f82191efb8,37523,1731640420593, state=OPEN 2024-11-15T03:13:49,056 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42051-0x1013c7bdd830000, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:49,056 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37523-0x1013c7bdd830002, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:49,056 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41451-0x1013c7bdd830003, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:49,056 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:49,056 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:49,056 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:49,057 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731640459057 2024-11-15T03:13:49,057 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-15T03:13:49,057 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T03:13:49,058 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-15T03:13:49,059 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-15T03:13:49,059 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:49,059 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T03:13:49,060 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T03:13:49,062 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5965b596ce7b87fabde47569d60c3f95, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,062 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6130297c45bd23340b368de2cc708c90, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,062 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5971f307349447e7f24b0dd05b96cffc, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,062 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1beb579fcf8e161a0b9c2051c671a8f2, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,063 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0f9e1e28ca8906f0f82f959a60c55ce, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,063 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4b611da4a73d4cfc99379713486e21c, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,063 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0eff1b91bc9b77579a5c44f1c985985, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,063 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48ff514f5498660a4e8fc753340bb847, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,063 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5219cfe492c74eb8312930d328964b6, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,063 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1704275babe02e36b29cfaadf5b81af, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,063 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb2ead3b463edc0f2b603da7d19f1201, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,063 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=860df8edb65f2fdd3c8bc00f89f94ba0, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,063 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eaf85cc654192e46d3ea0904493eea12, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,063 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a73d2d096e2f14a7e9365ac762927d55, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,063 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b96380a8d7110dffa8cd7844252025dd, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,063 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fac2a1b464a5ff2d581af6baef327bde, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,064 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f804d67a47ba185ffd6f6bc7b75464, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,064 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc930ecfca79abbf57ed28c20f79230f, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,064 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ef03d91ad777a77cf66f510e28419ba, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,064 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=21df709a703798d68290a0a63322957f, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,064 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1b8f1ea215e8c2f78186b80dafed1b28, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,064 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc93af816bdf94cf920a5adc31356783, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=5 2024-11-15T03:13:49,064 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=155ef591549501ad2da24d2c559fb7f9, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,064 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6cca2d3f6a001d96f34b7a93afec8fe7, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,064 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0bce1a97a9c7f320b315db443b47c23, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,064 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17e8b3b889a606b8c6826552e784d401, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,064 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-15T03:13:49,064 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731640489064 2024-11-15T03:13:49,065 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731640549064 2024-11-15T03:13:49,065 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 7 msec 2024-11-15T03:13:49,065 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T03:13:49,065 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T03:13:49,065 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T03:13:49,065 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T03:13:49,065 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T03:13:49,065 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T03:13:49,065 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T03:13:49,065 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T03:13:49,066 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-15T03:13:49,067 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-15T03:13:49,067 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T03:13:49,067 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T03:13:49,067 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T03:13:49,067 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:49,067 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T03:13:49,068 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T03:13:49,068 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-15T03:13:49,068 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=dc93af816bdf94cf920a5adc31356783, REOPEN/MOVE 2024-11-15T03:13:49,069 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true 2024-11-15T03:13:49,069 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:49,069 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:49,069 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T03:13:49,069 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=91f82191efb8,37523,1731640420593 2024-11-15T03:13:49,069 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:49,069 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:49,069 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:49,069 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:49,069 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:49,069 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:49,069 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.1731640422194, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:49,069 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-15T03:13:49,070 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=65 2024-11-15T03:13:49,070 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=91f82191efb8,37523,1731640420593, table=hbase:meta, region=1588230740 2024-11-15T03:13:49,070 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 91f82191efb8,37523,1731640420593, state=OPEN 2024-11-15T03:13:49,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41451-0x1013c7bdd830003, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:49,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42051-0x1013c7bdd830000, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:49,083 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37523-0x1013c7bdd830002, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:49,083 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:49,083 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:49,083 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:49,084 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731640459084 2024-11-15T03:13:49,084 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-15T03:13:49,084 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T03:13:49,085 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-15T03:13:49,086 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-15T03:13:49,086 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=91f82191efb8,37523,1731640420593 2024-11-15T03:13:49,086 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T03:13:49,086 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=91f82191efb8,37523,1731640420593 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T03:13:49,088 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5965b596ce7b87fabde47569d60c3f95, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,088 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6130297c45bd23340b368de2cc708c90, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,089 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5971f307349447e7f24b0dd05b96cffc, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,089 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1beb579fcf8e161a0b9c2051c671a8f2, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,089 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0f9e1e28ca8906f0f82f959a60c55ce, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,089 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4b611da4a73d4cfc99379713486e21c, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,089 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0eff1b91bc9b77579a5c44f1c985985, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,089 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48ff514f5498660a4e8fc753340bb847, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,089 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5219cfe492c74eb8312930d328964b6, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,089 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1704275babe02e36b29cfaadf5b81af, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,089 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb2ead3b463edc0f2b603da7d19f1201, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,089 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=860df8edb65f2fdd3c8bc00f89f94ba0, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,089 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eaf85cc654192e46d3ea0904493eea12, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,089 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a73d2d096e2f14a7e9365ac762927d55, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,089 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b96380a8d7110dffa8cd7844252025dd, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,089 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fac2a1b464a5ff2d581af6baef327bde, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,090 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f804d67a47ba185ffd6f6bc7b75464, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,090 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc930ecfca79abbf57ed28c20f79230f, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,090 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ef03d91ad777a77cf66f510e28419ba, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,090 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=21df709a703798d68290a0a63322957f, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,090 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1b8f1ea215e8c2f78186b80dafed1b28, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,090 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc93af816bdf94cf920a5adc31356783, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=5 2024-11-15T03:13:49,090 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=155ef591549501ad2da24d2c559fb7f9, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,090 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6cca2d3f6a001d96f34b7a93afec8fe7, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,090 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0bce1a97a9c7f320b315db443b47c23, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,090 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17e8b3b889a606b8c6826552e784d401, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,090 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-15T03:13:49,090 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731640489090 2024-11-15T03:13:49,090 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731640549090 2024-11-15T03:13:49,090 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 6 msec 2024-11-15T03:13:49,091 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T03:13:49,091 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T03:13:49,091 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T03:13:49,091 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T03:13:49,091 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T03:13:49,091 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T03:13:49,091 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T03:13:49,091 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T03:13:49,092 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-15T03:13:49,094 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-15T03:13:49,094 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T03:13:49,094 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T03:13:49,094 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:49,094 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T03:13:49,095 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T03:13:49,095 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-15T03:13:49,095 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=dc93af816bdf94cf920a5adc31356783, REOPEN/MOVE 2024-11-15T03:13:49,096 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42051 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T03:13:49,096 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true 2024-11-15T03:13:49,096 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:49,096 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:49,096 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T03:13:49,096 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=91f82191efb8,37523,1731640420593 2024-11-15T03:13:49,097 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:49,097 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:49,097 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:49,097 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42051 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T03:13:49,097 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:49,097 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:49,097 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-15T03:13:49,098 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=65 2024-11-15T03:13:49,098 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=91f82191efb8,37523,1731640420593, table=hbase:meta, region=1588230740 2024-11-15T03:13:49,098 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 91f82191efb8,37523,1731640420593, state=OPEN 2024-11-15T03:13:49,109 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37523-0x1013c7bdd830002, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:49,109 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41451-0x1013c7bdd830003, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:49,109 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42051-0x1013c7bdd830000, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:49,109 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:49,109 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:49,109 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:49,110 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731640459110 2024-11-15T03:13:49,110 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-15T03:13:49,110 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T03:13:49,111 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-15T03:13:49,112 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-15T03:13:49,112 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T03:13:49,112 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T03:13:49,112 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T03:13:49,114 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5965b596ce7b87fabde47569d60c3f95, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,114 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6130297c45bd23340b368de2cc708c90, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,114 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5971f307349447e7f24b0dd05b96cffc, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,114 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1beb579fcf8e161a0b9c2051c671a8f2, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,115 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0f9e1e28ca8906f0f82f959a60c55ce, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,115 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4b611da4a73d4cfc99379713486e21c, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,115 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0eff1b91bc9b77579a5c44f1c985985, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,115 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48ff514f5498660a4e8fc753340bb847, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,115 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5219cfe492c74eb8312930d328964b6, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,115 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1704275babe02e36b29cfaadf5b81af, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,115 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb2ead3b463edc0f2b603da7d19f1201, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,115 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=860df8edb65f2fdd3c8bc00f89f94ba0, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,115 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eaf85cc654192e46d3ea0904493eea12, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,115 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a73d2d096e2f14a7e9365ac762927d55, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,115 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b96380a8d7110dffa8cd7844252025dd, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,115 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fac2a1b464a5ff2d581af6baef327bde, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,116 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f804d67a47ba185ffd6f6bc7b75464, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,116 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc930ecfca79abbf57ed28c20f79230f, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,116 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ef03d91ad777a77cf66f510e28419ba, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,116 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=21df709a703798d68290a0a63322957f, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,116 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1b8f1ea215e8c2f78186b80dafed1b28, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,116 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc93af816bdf94cf920a5adc31356783, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=5 2024-11-15T03:13:49,116 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=155ef591549501ad2da24d2c559fb7f9, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,116 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6cca2d3f6a001d96f34b7a93afec8fe7, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,116 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0bce1a97a9c7f320b315db443b47c23, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,116 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17e8b3b889a606b8c6826552e784d401, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,116 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-15T03:13:49,116 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731640489116 2024-11-15T03:13:49,116 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731640549116 2024-11-15T03:13:49,116 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 6 msec 2024-11-15T03:13:49,116 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T03:13:49,116 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T03:13:49,116 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T03:13:49,117 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T03:13:49,117 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T03:13:49,117 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T03:13:49,117 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T03:13:49,117 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T03:13:49,118 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-15T03:13:49,119 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-15T03:13:49,119 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T03:13:49,119 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T03:13:49,119 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T03:13:49,119 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T03:13:49,119 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-15T03:13:49,119 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=dc93af816bdf94cf920a5adc31356783, REOPEN/MOVE 2024-11-15T03:13:49,120 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true 2024-11-15T03:13:49,120 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:49,120 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:49,120 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T03:13:49,120 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:49,120 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:49,120 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:49,120 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:49,120 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-15T03:13:49,121 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=65 2024-11-15T03:13:49,121 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=91f82191efb8,37523,1731640420593, table=hbase:meta, region=1588230740 2024-11-15T03:13:49,121 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 91f82191efb8,37523,1731640420593, state=OPEN 2024-11-15T03:13:49,130 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37523-0x1013c7bdd830002, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:49,130 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42051-0x1013c7bdd830000, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:49,130 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41451-0x1013c7bdd830003, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:49,130 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:49,130 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:49,130 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:49,131 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731640459131 2024-11-15T03:13:49,131 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-15T03:13:49,131 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T03:13:49,132 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-15T03:13:49,132 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-15T03:13:49,132 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:49,132 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T03:13:49,133 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T03:13:49,135 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5965b596ce7b87fabde47569d60c3f95, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,135 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6130297c45bd23340b368de2cc708c90, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,135 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5971f307349447e7f24b0dd05b96cffc, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,135 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1beb579fcf8e161a0b9c2051c671a8f2, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,135 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0f9e1e28ca8906f0f82f959a60c55ce, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,135 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4b611da4a73d4cfc99379713486e21c, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,135 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0eff1b91bc9b77579a5c44f1c985985, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,135 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48ff514f5498660a4e8fc753340bb847, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,135 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5219cfe492c74eb8312930d328964b6, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,135 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1704275babe02e36b29cfaadf5b81af, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,135 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb2ead3b463edc0f2b603da7d19f1201, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,135 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=860df8edb65f2fdd3c8bc00f89f94ba0, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,135 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eaf85cc654192e46d3ea0904493eea12, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,135 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a73d2d096e2f14a7e9365ac762927d55, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,136 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b96380a8d7110dffa8cd7844252025dd, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,136 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fac2a1b464a5ff2d581af6baef327bde, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,136 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f804d67a47ba185ffd6f6bc7b75464, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,136 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc930ecfca79abbf57ed28c20f79230f, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,136 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ef03d91ad777a77cf66f510e28419ba, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,136 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=21df709a703798d68290a0a63322957f, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,136 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1b8f1ea215e8c2f78186b80dafed1b28, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,136 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc93af816bdf94cf920a5adc31356783, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=5 2024-11-15T03:13:49,136 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=155ef591549501ad2da24d2c559fb7f9, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,136 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6cca2d3f6a001d96f34b7a93afec8fe7, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,136 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0bce1a97a9c7f320b315db443b47c23, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,136 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17e8b3b889a606b8c6826552e784d401, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,136 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-15T03:13:49,136 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731640489136 2024-11-15T03:13:49,137 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731640549137 2024-11-15T03:13:49,137 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 5 msec 2024-11-15T03:13:49,137 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T03:13:49,137 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T03:13:49,137 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T03:13:49,137 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T03:13:49,137 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T03:13:49,137 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T03:13:49,137 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T03:13:49,137 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T03:13:49,138 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-15T03:13:49,139 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-15T03:13:49,139 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T03:13:49,139 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T03:13:49,139 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T03:13:49,140 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-15T03:13:49,140 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=dc93af816bdf94cf920a5adc31356783, REOPEN/MOVE 2024-11-15T03:13:49,140 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true 2024-11-15T03:13:49,140 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:49,140 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:49,140 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:49,140 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:49,140 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:49,141 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-15T03:13:49,141 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=65 2024-11-15T03:13:49,141 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=91f82191efb8,37523,1731640420593, table=hbase:meta, region=1588230740 2024-11-15T03:13:49,141 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 91f82191efb8,37523,1731640420593, state=OPEN 2024-11-15T03:13:49,151 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42051-0x1013c7bdd830000, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:49,151 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41451-0x1013c7bdd830003, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:49,151 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37523-0x1013c7bdd830002, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:49,151 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:49,151 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:49,151 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:49,152 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731640459152 2024-11-15T03:13:49,152 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-15T03:13:49,152 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T03:13:49,153 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-15T03:13:49,157 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5965b596ce7b87fabde47569d60c3f95, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,157 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-15T03:13:49,157 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6130297c45bd23340b368de2cc708c90, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,157 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:49,157 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T03:13:49,157 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5971f307349447e7f24b0dd05b96cffc, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,158 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1beb579fcf8e161a0b9c2051c671a8f2, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,158 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T03:13:49,158 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0f9e1e28ca8906f0f82f959a60c55ce, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,158 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4b611da4a73d4cfc99379713486e21c, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,158 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0eff1b91bc9b77579a5c44f1c985985, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,158 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48ff514f5498660a4e8fc753340bb847, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,158 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5219cfe492c74eb8312930d328964b6, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1704275babe02e36b29cfaadf5b81af, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb2ead3b463edc0f2b603da7d19f1201, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=860df8edb65f2fdd3c8bc00f89f94ba0, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eaf85cc654192e46d3ea0904493eea12, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a73d2d096e2f14a7e9365ac762927d55, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b96380a8d7110dffa8cd7844252025dd, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fac2a1b464a5ff2d581af6baef327bde, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f804d67a47ba185ffd6f6bc7b75464, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc930ecfca79abbf57ed28c20f79230f, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ef03d91ad777a77cf66f510e28419ba, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=21df709a703798d68290a0a63322957f, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1b8f1ea215e8c2f78186b80dafed1b28, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc93af816bdf94cf920a5adc31356783, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=5 2024-11-15T03:13:49,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=155ef591549501ad2da24d2c559fb7f9, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6cca2d3f6a001d96f34b7a93afec8fe7, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,161 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0bce1a97a9c7f320b315db443b47c23, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,161 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17e8b3b889a606b8c6826552e784d401, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,161 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-15T03:13:49,161 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731640489161 2024-11-15T03:13:49,161 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731640549161 2024-11-15T03:13:49,161 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-15T03:13:49,161 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T03:13:49,161 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T03:13:49,161 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T03:13:49,161 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T03:13:49,161 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T03:13:49,161 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T03:13:49,161 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T03:13:49,162 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T03:13:49,162 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-15T03:13:49,164 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-15T03:13:49,164 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T03:13:49,164 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T03:13:49,165 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-15T03:13:49,165 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=dc93af816bdf94cf920a5adc31356783, REOPEN/MOVE 2024-11-15T03:13:49,165 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true 2024-11-15T03:13:49,165 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 2024-11-15T03:13:49,166 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:49,166 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 91f82191efb8%2C42027%2C1731640420387.meta.1731640422710.meta, worker=91f82191efb8,37523,1731640420593 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T03:13:49,166 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-15T03:13:49,166 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=65 2024-11-15T03:13:49,167 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=91f82191efb8,37523,1731640420593, table=hbase:meta, region=1588230740 2024-11-15T03:13:49,167 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 91f82191efb8,37523,1731640420593, state=OPEN 2024-11-15T03:13:49,177 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37523-0x1013c7bdd830002, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:49,177 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41451-0x1013c7bdd830003, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:49,177 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42051-0x1013c7bdd830000, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:49,177 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:49,177 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:49,177 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:49,178 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731640459178 2024-11-15T03:13:49,178 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-15T03:13:49,178 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T03:13:49,179 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-15T03:13:49,180 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-15T03:13:49,181 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true 2024-11-15T03:13:49,181 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T03:13:49,181 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1604) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T03:13:49,183 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5965b596ce7b87fabde47569d60c3f95, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,183 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6130297c45bd23340b368de2cc708c90, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,184 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5971f307349447e7f24b0dd05b96cffc, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,184 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1beb579fcf8e161a0b9c2051c671a8f2, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,184 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0f9e1e28ca8906f0f82f959a60c55ce, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,184 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4b611da4a73d4cfc99379713486e21c, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,184 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0eff1b91bc9b77579a5c44f1c985985, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,184 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48ff514f5498660a4e8fc753340bb847, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,184 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5219cfe492c74eb8312930d328964b6, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,184 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1704275babe02e36b29cfaadf5b81af, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,184 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb2ead3b463edc0f2b603da7d19f1201, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,185 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=860df8edb65f2fdd3c8bc00f89f94ba0, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,185 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eaf85cc654192e46d3ea0904493eea12, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,185 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a73d2d096e2f14a7e9365ac762927d55, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,185 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b96380a8d7110dffa8cd7844252025dd, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,185 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fac2a1b464a5ff2d581af6baef327bde, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,185 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f804d67a47ba185ffd6f6bc7b75464, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,185 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc930ecfca79abbf57ed28c20f79230f, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,185 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ef03d91ad777a77cf66f510e28419ba, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,185 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=21df709a703798d68290a0a63322957f, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,185 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1b8f1ea215e8c2f78186b80dafed1b28, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,186 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc93af816bdf94cf920a5adc31356783, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=5 2024-11-15T03:13:49,186 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=155ef591549501ad2da24d2c559fb7f9, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,186 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6cca2d3f6a001d96f34b7a93afec8fe7, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,186 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0bce1a97a9c7f320b315db443b47c23, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,186 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17e8b3b889a606b8c6826552e784d401, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,186 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-15T03:13:49,186 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731640489186 2024-11-15T03:13:49,186 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731640549186 2024-11-15T03:13:49,186 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 7 msec 2024-11-15T03:13:49,186 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T03:13:49,186 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T03:13:49,186 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T03:13:49,186 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T03:13:49,187 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T03:13:49,187 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T03:13:49,187 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T03:13:49,187 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T03:13:49,188 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-15T03:13:49,190 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-15T03:13:49,190 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T03:13:49,190 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-15T03:13:49,190 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=dc93af816bdf94cf920a5adc31356783, REOPEN/MOVE 2024-11-15T03:13:49,191 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true 2024-11-15T03:13:49,191 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-15T03:13:49,191 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-15T03:13:49,192 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=65 2024-11-15T03:13:49,192 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=91f82191efb8,37523,1731640420593, table=hbase:meta, region=1588230740 2024-11-15T03:13:49,192 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 91f82191efb8,37523,1731640420593, state=OPEN 2024-11-15T03:13:49,199 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42051 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T03:13:49,200 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42051 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T03:13:49,203 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41451-0x1013c7bdd830003, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:49,203 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37523-0x1013c7bdd830002, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:49,203 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42051-0x1013c7bdd830000, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T03:13:49,204 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:49,204 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:49,204 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T03:13:49,204 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731640459204 2024-11-15T03:13:49,204 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-15T03:13:49,204 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T03:13:49,205 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-15T03:13:49,206 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=60, state=ROLLEDBACK, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bb2ead3b463edc0f2b603da7d19f1201, server=91f82191efb8,41451,1731640420638:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 91f82191efb8,42027,1731640420387, splitWal=true, meta=true exec-time=2.5320 sec 2024-11-15T03:13:49,209 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5965b596ce7b87fabde47569d60c3f95, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,209 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6130297c45bd23340b368de2cc708c90, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,209 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5971f307349447e7f24b0dd05b96cffc, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,209 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1beb579fcf8e161a0b9c2051c671a8f2, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,209 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0f9e1e28ca8906f0f82f959a60c55ce, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,209 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4b611da4a73d4cfc99379713486e21c, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,209 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0eff1b91bc9b77579a5c44f1c985985, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,209 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48ff514f5498660a4e8fc753340bb847, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,210 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5219cfe492c74eb8312930d328964b6, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,210 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1704275babe02e36b29cfaadf5b81af, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,210 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb2ead3b463edc0f2b603da7d19f1201, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,210 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=860df8edb65f2fdd3c8bc00f89f94ba0, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,210 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eaf85cc654192e46d3ea0904493eea12, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,210 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a73d2d096e2f14a7e9365ac762927d55, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,210 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b96380a8d7110dffa8cd7844252025dd, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,210 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fac2a1b464a5ff2d581af6baef327bde, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,210 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f804d67a47ba185ffd6f6bc7b75464, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,210 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc930ecfca79abbf57ed28c20f79230f, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,210 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ef03d91ad777a77cf66f510e28419ba, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,210 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=21df709a703798d68290a0a63322957f, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,211 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1b8f1ea215e8c2f78186b80dafed1b28, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,211 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc93af816bdf94cf920a5adc31356783, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=5 2024-11-15T03:13:49,211 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=155ef591549501ad2da24d2c559fb7f9, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,211 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6cca2d3f6a001d96f34b7a93afec8fe7, regionState=OPEN, lastHost=91f82191efb8,41451,1731640420638, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,211 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0bce1a97a9c7f320b315db443b47c23, regionState=OPENING, lastHost=91f82191efb8,42027,1731640420387, regionLocation=91f82191efb8,41451,1731640420638, openSeqNum=2 2024-11-15T03:13:49,211 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17e8b3b889a606b8c6826552e784d401, regionState=OPEN, lastHost=91f82191efb8,37523,1731640420593, regionLocation=91f82191efb8,37523,1731640420593, openSeqNum=2 2024-11-15T03:13:49,211 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-15T03:13:49,211 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731640489211 2024-11-15T03:13:49,211 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731640549211 2024-11-15T03:13:49,211 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 6 msec 2024-11-15T03:13:49,227 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.assignment.TestRollbackSCP#testFailAndRollback Thread=351 (was 384), OpenFileDescriptor=628 (was 650), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=277 (was 277), ProcessCount=11 (was 11), AvailableMemoryMB=2962 (was 2998) 2024-11-15T03:13:49,227 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-15T03:13:49,227 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T03:13:49,227 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T03:13:49,227 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T03:13:49,228 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T03:13:49,228 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T03:13:49,228 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-15T03:13:49,228 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=688211475, stopped=false 2024-11-15T03:13:49,228 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=91f82191efb8,42051,1731640419585 2024-11-15T03:13:49,240 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37523-0x1013c7bdd830002, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-15T03:13:49,240 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41451-0x1013c7bdd830003, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-15T03:13:49,240 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42051-0x1013c7bdd830000, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-15T03:13:49,240 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41451-0x1013c7bdd830003, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T03:13:49,240 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37523-0x1013c7bdd830002, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T03:13:49,240 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T03:13:49,241 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T03:13:49,241 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T03:13:49,241 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T03:13:49,241 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42051-0x1013c7bdd830000, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T03:13:49,241 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '91f82191efb8,37523,1731640420593' ***** 2024-11-15T03:13:49,241 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41451-0x1013c7bdd830003, quorum=127.0.0.1:64070, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T03:13:49,241 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-15T03:13:49,241 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37523-0x1013c7bdd830002, quorum=127.0.0.1:64070, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T03:13:49,241 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '91f82191efb8,41451,1731640420638' ***** 2024-11-15T03:13:49,241 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-15T03:13:49,241 INFO [RS:1;91f82191efb8:37523 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-15T03:13:49,241 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42051-0x1013c7bdd830000, quorum=127.0.0.1:64070, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T03:13:49,241 INFO [RS:1;91f82191efb8:37523 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-15T03:13:49,241 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-15T03:13:49,242 INFO [RS:1;91f82191efb8:37523 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-15T03:13:49,242 INFO [RS:1;91f82191efb8:37523 {}] regionserver.HRegionServer(3091): Received CLOSE for 17e8b3b889a606b8c6826552e784d401 2024-11-15T03:13:49,242 INFO [RS:2;91f82191efb8:41451 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-15T03:13:49,242 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-15T03:13:49,242 INFO [RS:2;91f82191efb8:41451 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-15T03:13:49,242 INFO [RS:2;91f82191efb8:41451 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-15T03:13:49,242 INFO [RS:2;91f82191efb8:41451 {}] regionserver.HRegionServer(3091): Received CLOSE for 5965b596ce7b87fabde47569d60c3f95 2024-11-15T03:13:49,242 INFO [RS:1;91f82191efb8:37523 {}] regionserver.HRegionServer(3091): Received CLOSE for fac2a1b464a5ff2d581af6baef327bde 2024-11-15T03:13:49,242 INFO [RS:1;91f82191efb8:37523 {}] regionserver.HRegionServer(3091): Received CLOSE for 21df709a703798d68290a0a63322957f 2024-11-15T03:13:49,242 INFO [RS:1;91f82191efb8:37523 {}] regionserver.HRegionServer(3091): Received CLOSE for 6130297c45bd23340b368de2cc708c90 2024-11-15T03:13:49,242 INFO [RS:1;91f82191efb8:37523 {}] regionserver.HRegionServer(3091): Received CLOSE for b1704275babe02e36b29cfaadf5b81af 2024-11-15T03:13:49,242 INFO [RS:2;91f82191efb8:41451 {}] regionserver.HRegionServer(3091): Received CLOSE for 6cca2d3f6a001d96f34b7a93afec8fe7 2024-11-15T03:13:49,242 INFO [RS:1;91f82191efb8:37523 {}] regionserver.HRegionServer(3091): Received CLOSE for 48ff514f5498660a4e8fc753340bb847 2024-11-15T03:13:49,242 INFO [RS:2;91f82191efb8:41451 {}] regionserver.HRegionServer(3091): Received CLOSE for bb2ead3b463edc0f2b603da7d19f1201 2024-11-15T03:13:49,242 INFO [RS:1;91f82191efb8:37523 {}] regionserver.HRegionServer(3091): Received CLOSE for 155ef591549501ad2da24d2c559fb7f9 2024-11-15T03:13:49,243 INFO [RS:2;91f82191efb8:41451 {}] regionserver.HRegionServer(3091): Received CLOSE for e0bce1a97a9c7f320b315db443b47c23 2024-11-15T03:13:49,243 INFO [RS:1;91f82191efb8:37523 {}] regionserver.HRegionServer(3091): Received CLOSE for 5971f307349447e7f24b0dd05b96cffc 2024-11-15T03:13:49,243 INFO [RS:2;91f82191efb8:41451 {}] regionserver.HRegionServer(3091): Received CLOSE for b96380a8d7110dffa8cd7844252025dd 2024-11-15T03:13:49,243 INFO [RS:1;91f82191efb8:37523 {}] regionserver.HRegionServer(3091): Received CLOSE for eaf85cc654192e46d3ea0904493eea12 2024-11-15T03:13:49,243 INFO [RS:2;91f82191efb8:41451 {}] regionserver.HRegionServer(3091): Received CLOSE for 23f804d67a47ba185ffd6f6bc7b75464 2024-11-15T03:13:49,243 INFO [RS:1;91f82191efb8:37523 {}] regionserver.HRegionServer(3091): Received CLOSE for c4b611da4a73d4cfc99379713486e21c 2024-11-15T03:13:49,243 INFO [RS:2;91f82191efb8:41451 {}] regionserver.HRegionServer(3091): Received CLOSE for 860df8edb65f2fdd3c8bc00f89f94ba0 2024-11-15T03:13:49,243 INFO [RS:2;91f82191efb8:41451 {}] regionserver.HRegionServer(3091): Received CLOSE for e0f9e1e28ca8906f0f82f959a60c55ce 2024-11-15T03:13:49,243 INFO [RS:2;91f82191efb8:41451 {}] regionserver.HRegionServer(3091): Received CLOSE for f5219cfe492c74eb8312930d328964b6 2024-11-15T03:13:49,243 INFO [RS:1;91f82191efb8:37523 {}] regionserver.HRegionServer(959): stopping server 91f82191efb8,37523,1731640420593 2024-11-15T03:13:49,243 INFO [RS:2;91f82191efb8:41451 {}] regionserver.HRegionServer(3091): Received CLOSE for dc93af816bdf94cf920a5adc31356783 2024-11-15T03:13:49,243 INFO [RS:1;91f82191efb8:37523 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-15T03:13:49,243 INFO [RS:2;91f82191efb8:41451 {}] regionserver.HRegionServer(3091): Received CLOSE for 1beb579fcf8e161a0b9c2051c671a8f2 2024-11-15T03:13:49,243 INFO [RS:2;91f82191efb8:41451 {}] regionserver.HRegionServer(3091): Received CLOSE for dc930ecfca79abbf57ed28c20f79230f 2024-11-15T03:13:49,243 INFO [RS:1;91f82191efb8:37523 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;91f82191efb8:37523. 2024-11-15T03:13:49,243 INFO [RS:2;91f82191efb8:41451 {}] regionserver.HRegionServer(3091): Received CLOSE for 4ef03d91ad777a77cf66f510e28419ba 2024-11-15T03:13:49,243 INFO [RS:2;91f82191efb8:41451 {}] regionserver.HRegionServer(959): stopping server 91f82191efb8,41451,1731640420638 2024-11-15T03:13:49,243 DEBUG [RS:1;91f82191efb8:37523 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T03:13:49,243 INFO [RS:2;91f82191efb8:41451 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-15T03:13:49,243 DEBUG [RS:1;91f82191efb8:37523 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T03:13:49,243 INFO [RS:2;91f82191efb8:41451 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;91f82191efb8:41451. 2024-11-15T03:13:49,243 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 17e8b3b889a606b8c6826552e784d401, disabling compactions & flushes 2024-11-15T03:13:49,243 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5965b596ce7b87fabde47569d60c3f95, disabling compactions & flushes 2024-11-15T03:13:49,243 DEBUG [RS:2;91f82191efb8:41451 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T03:13:49,243 DEBUG [RS:2;91f82191efb8:41451 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T03:13:49,243 INFO [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,zzz,1731640423069.17e8b3b889a606b8c6826552e784d401. 2024-11-15T03:13:49,243 INFO [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,,1731640423069.5965b596ce7b87fabde47569d60c3f95. 2024-11-15T03:13:49,243 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1731640423069.17e8b3b889a606b8c6826552e784d401. 2024-11-15T03:13:49,243 INFO [RS:1;91f82191efb8:37523 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-15T03:13:49,243 INFO [RS:1;91f82191efb8:37523 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-15T03:13:49,243 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1731640423069.17e8b3b889a606b8c6826552e784d401. after waiting 0 ms 2024-11-15T03:13:49,243 INFO [RS:2;91f82191efb8:41451 {}] regionserver.HRegionServer(1321): Waiting on 13 regions to close 2024-11-15T03:13:49,243 INFO [RS:1;91f82191efb8:37523 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-15T03:13:49,243 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1731640423069.17e8b3b889a606b8c6826552e784d401. 2024-11-15T03:13:49,243 INFO [RS:1;91f82191efb8:37523 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-15T03:13:49,243 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1731640423069.5965b596ce7b87fabde47569d60c3f95. 2024-11-15T03:13:49,244 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,,1731640423069.5965b596ce7b87fabde47569d60c3f95. after waiting 0 ms 2024-11-15T03:13:49,244 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,,1731640423069.5965b596ce7b87fabde47569d60c3f95. 2024-11-15T03:13:49,244 DEBUG [RS:2;91f82191efb8:41451 {}] regionserver.HRegionServer(1351): Waiting on 1beb579fcf8e161a0b9c2051c671a8f2, 23f804d67a47ba185ffd6f6bc7b75464, 4ef03d91ad777a77cf66f510e28419ba, 5965b596ce7b87fabde47569d60c3f95, 6cca2d3f6a001d96f34b7a93afec8fe7, 860df8edb65f2fdd3c8bc00f89f94ba0, b96380a8d7110dffa8cd7844252025dd, bb2ead3b463edc0f2b603da7d19f1201, dc930ecfca79abbf57ed28c20f79230f, dc93af816bdf94cf920a5adc31356783, e0bce1a97a9c7f320b315db443b47c23, e0f9e1e28ca8906f0f82f959a60c55ce, f5219cfe492c74eb8312930d328964b6 2024-11-15T03:13:49,244 INFO [RS:1;91f82191efb8:37523 {}] regionserver.HRegionServer(1321): Waiting on 11 regions to close 2024-11-15T03:13:49,244 DEBUG [RS:1;91f82191efb8:37523 {}] regionserver.HRegionServer(1351): Waiting on 155ef591549501ad2da24d2c559fb7f9, 1588230740, 17e8b3b889a606b8c6826552e784d401, 1b8f1ea215e8c2f78186b80dafed1b28, 21df709a703798d68290a0a63322957f, 48ff514f5498660a4e8fc753340bb847, 5971f307349447e7f24b0dd05b96cffc, 6130297c45bd23340b368de2cc708c90, b1704275babe02e36b29cfaadf5b81af, c4b611da4a73d4cfc99379713486e21c, eaf85cc654192e46d3ea0904493eea12, fac2a1b464a5ff2d581af6baef327bde 2024-11-15T03:13:49,244 DEBUG [RS_CLOSE_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-15T03:13:49,244 INFO [RS_CLOSE_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-15T03:13:49,244 DEBUG [RS_CLOSE_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-15T03:13:49,244 DEBUG [RS_CLOSE_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-15T03:13:49,244 DEBUG [RS_CLOSE_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-15T03:13:49,244 INFO [RS_CLOSE_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.66 KB heapSize=5.44 KB 2024-11-15T03:13:49,249 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/17e8b3b889a606b8c6826552e784d401/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T03:13:49,250 INFO [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,zzz,1731640423069.17e8b3b889a606b8c6826552e784d401. 2024-11-15T03:13:49,250 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 17e8b3b889a606b8c6826552e784d401: Waiting for close lock at 1731640429242Running coprocessor pre-close hooks at 1731640429243 (+1 ms)Disabling compacts and flushes for region at 1731640429243Disabling writes for close at 1731640429243Writing region close event to WAL at 1731640429244 (+1 ms)Running coprocessor post-close hooks at 1731640429250 (+6 ms)Closed at 1731640429250 2024-11-15T03:13:49,250 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,zzz,1731640423069.17e8b3b889a606b8c6826552e784d401. 2024-11-15T03:13:49,251 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fac2a1b464a5ff2d581af6baef327bde, disabling compactions & flushes 2024-11-15T03:13:49,251 INFO [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ppp,1731640423069.fac2a1b464a5ff2d581af6baef327bde. 2024-11-15T03:13:49,251 INFO [regionserver/91f82191efb8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T03:13:49,251 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1731640423069.fac2a1b464a5ff2d581af6baef327bde. 2024-11-15T03:13:49,251 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1731640423069.fac2a1b464a5ff2d581af6baef327bde. after waiting 0 ms 2024-11-15T03:13:49,251 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1731640423069.fac2a1b464a5ff2d581af6baef327bde. 2024-11-15T03:13:49,251 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/5965b596ce7b87fabde47569d60c3f95/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T03:13:49,252 INFO [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,,1731640423069.5965b596ce7b87fabde47569d60c3f95. 2024-11-15T03:13:49,252 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5965b596ce7b87fabde47569d60c3f95: Waiting for close lock at 1731640429243Running coprocessor pre-close hooks at 1731640429243Disabling compacts and flushes for region at 1731640429243Disabling writes for close at 1731640429244 (+1 ms)Writing region close event to WAL at 1731640429244Running coprocessor post-close hooks at 1731640429252 (+8 ms)Closed at 1731640429252 2024-11-15T03:13:49,252 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,,1731640423069.5965b596ce7b87fabde47569d60c3f95. 2024-11-15T03:13:49,253 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6cca2d3f6a001d96f34b7a93afec8fe7, disabling compactions & flushes 2024-11-15T03:13:49,253 INFO [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,xxx,1731640423069.6cca2d3f6a001d96f34b7a93afec8fe7. 2024-11-15T03:13:49,253 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1731640423069.6cca2d3f6a001d96f34b7a93afec8fe7. 2024-11-15T03:13:49,253 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1731640423069.6cca2d3f6a001d96f34b7a93afec8fe7. after waiting 0 ms 2024-11-15T03:13:49,253 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1731640423069.6cca2d3f6a001d96f34b7a93afec8fe7. 2024-11-15T03:13:49,256 INFO [regionserver/91f82191efb8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T03:13:49,258 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/6cca2d3f6a001d96f34b7a93afec8fe7/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T03:13:49,259 INFO [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,xxx,1731640423069.6cca2d3f6a001d96f34b7a93afec8fe7. 2024-11-15T03:13:49,259 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6cca2d3f6a001d96f34b7a93afec8fe7: Waiting for close lock at 1731640429253Running coprocessor pre-close hooks at 1731640429253Disabling compacts and flushes for region at 1731640429253Disabling writes for close at 1731640429253Writing region close event to WAL at 1731640429253Running coprocessor post-close hooks at 1731640429259 (+6 ms)Closed at 1731640429259 2024-11-15T03:13:49,259 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,xxx,1731640423069.6cca2d3f6a001d96f34b7a93afec8fe7. 2024-11-15T03:13:49,259 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing bb2ead3b463edc0f2b603da7d19f1201, disabling compactions & flushes 2024-11-15T03:13:49,259 INFO [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,kkk,1731640423069.bb2ead3b463edc0f2b603da7d19f1201. 2024-11-15T03:13:49,259 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1731640423069.bb2ead3b463edc0f2b603da7d19f1201. 2024-11-15T03:13:49,259 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1731640423069.bb2ead3b463edc0f2b603da7d19f1201. after waiting 0 ms 2024-11-15T03:13:49,259 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1731640423069.bb2ead3b463edc0f2b603da7d19f1201. 2024-11-15T03:13:49,264 DEBUG [RS_CLOSE_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/hbase/meta/1588230740/.tmp/info/a141ab00a364450599fca9e117c8a708 is 138, key is test,ccc,1731640423069.5971f307349447e7f24b0dd05b96cffc./info:regioninfo/1731640428081/Put/seqid=0 2024-11-15T03:13:49,266 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/fac2a1b464a5ff2d581af6baef327bde/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T03:13:49,267 INFO [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ppp,1731640423069.fac2a1b464a5ff2d581af6baef327bde. 2024-11-15T03:13:49,267 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fac2a1b464a5ff2d581af6baef327bde: Waiting for close lock at 1731640429250Running coprocessor pre-close hooks at 1731640429250Disabling compacts and flushes for region at 1731640429251 (+1 ms)Disabling writes for close at 1731640429251Writing region close event to WAL at 1731640429253 (+2 ms)Running coprocessor post-close hooks at 1731640429267 (+14 ms)Closed at 1731640429267 2024-11-15T03:13:49,267 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ppp,1731640423069.fac2a1b464a5ff2d581af6baef327bde. 2024-11-15T03:13:49,267 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 21df709a703798d68290a0a63322957f, disabling compactions & flushes 2024-11-15T03:13:49,267 INFO [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ttt,1731640423069.21df709a703798d68290a0a63322957f. 2024-11-15T03:13:49,267 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1731640423069.21df709a703798d68290a0a63322957f. 2024-11-15T03:13:49,267 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1731640423069.21df709a703798d68290a0a63322957f. after waiting 0 ms 2024-11-15T03:13:49,267 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1731640423069.21df709a703798d68290a0a63322957f. 2024-11-15T03:13:49,273 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/bb2ead3b463edc0f2b603da7d19f1201/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T03:13:49,274 INFO [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,kkk,1731640423069.bb2ead3b463edc0f2b603da7d19f1201. 2024-11-15T03:13:49,274 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for bb2ead3b463edc0f2b603da7d19f1201: Waiting for close lock at 1731640429259Running coprocessor pre-close hooks at 1731640429259Disabling compacts and flushes for region at 1731640429259Disabling writes for close at 1731640429259Writing region close event to WAL at 1731640429264 (+5 ms)Running coprocessor post-close hooks at 1731640429274 (+10 ms)Closed at 1731640429274 2024-11-15T03:13:49,274 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,kkk,1731640423069.bb2ead3b463edc0f2b603da7d19f1201. 2024-11-15T03:13:49,274 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e0bce1a97a9c7f320b315db443b47c23, disabling compactions & flushes 2024-11-15T03:13:49,274 INFO [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,yyy,1731640423069.e0bce1a97a9c7f320b315db443b47c23. 2024-11-15T03:13:49,274 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1731640423069.e0bce1a97a9c7f320b315db443b47c23. 2024-11-15T03:13:49,274 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1731640423069.e0bce1a97a9c7f320b315db443b47c23. after waiting 0 ms 2024-11-15T03:13:49,274 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1731640423069.e0bce1a97a9c7f320b315db443b47c23. 2024-11-15T03:13:49,277 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/21df709a703798d68290a0a63322957f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T03:13:49,278 INFO [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ttt,1731640423069.21df709a703798d68290a0a63322957f. 2024-11-15T03:13:49,278 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 21df709a703798d68290a0a63322957f: Waiting for close lock at 1731640429267Running coprocessor pre-close hooks at 1731640429267Disabling compacts and flushes for region at 1731640429267Disabling writes for close at 1731640429267Writing region close event to WAL at 1731640429268 (+1 ms)Running coprocessor post-close hooks at 1731640429278 (+10 ms)Closed at 1731640429278 2024-11-15T03:13:49,278 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ttt,1731640423069.21df709a703798d68290a0a63322957f. 2024-11-15T03:13:49,279 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6130297c45bd23340b368de2cc708c90, disabling compactions & flushes 2024-11-15T03:13:49,279 INFO [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,bbb,1731640423069.6130297c45bd23340b368de2cc708c90. 2024-11-15T03:13:49,279 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1731640423069.6130297c45bd23340b368de2cc708c90. 2024-11-15T03:13:49,279 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1731640423069.6130297c45bd23340b368de2cc708c90. after waiting 0 ms 2024-11-15T03:13:49,279 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1731640423069.6130297c45bd23340b368de2cc708c90. 2024-11-15T03:13:49,285 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/e0bce1a97a9c7f320b315db443b47c23/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T03:13:49,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36559 is added to blk_1073741869_1045 (size=8195) 2024-11-15T03:13:49,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38319 is added to blk_1073741869_1045 (size=8195) 2024-11-15T03:13:49,287 INFO [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,yyy,1731640423069.e0bce1a97a9c7f320b315db443b47c23. 2024-11-15T03:13:49,287 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e0bce1a97a9c7f320b315db443b47c23: Waiting for close lock at 1731640429274Running coprocessor pre-close hooks at 1731640429274Disabling compacts and flushes for region at 1731640429274Disabling writes for close at 1731640429274Writing region close event to WAL at 1731640429277 (+3 ms)Running coprocessor post-close hooks at 1731640429287 (+10 ms)Closed at 1731640429287 2024-11-15T03:13:49,287 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,yyy,1731640423069.e0bce1a97a9c7f320b315db443b47c23. 2024-11-15T03:13:49,287 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b96380a8d7110dffa8cd7844252025dd, disabling compactions & flushes 2024-11-15T03:13:49,287 INFO [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ooo,1731640423069.b96380a8d7110dffa8cd7844252025dd. 2024-11-15T03:13:49,287 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1731640423069.b96380a8d7110dffa8cd7844252025dd. 2024-11-15T03:13:49,287 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1731640423069.b96380a8d7110dffa8cd7844252025dd. after waiting 0 ms 2024-11-15T03:13:49,288 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1731640423069.b96380a8d7110dffa8cd7844252025dd. 2024-11-15T03:13:49,289 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/6130297c45bd23340b368de2cc708c90/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T03:13:49,291 INFO [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,bbb,1731640423069.6130297c45bd23340b368de2cc708c90. 2024-11-15T03:13:49,291 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6130297c45bd23340b368de2cc708c90: Waiting for close lock at 1731640429278Running coprocessor pre-close hooks at 1731640429278Disabling compacts and flushes for region at 1731640429278Disabling writes for close at 1731640429279 (+1 ms)Writing region close event to WAL at 1731640429284 (+5 ms)Running coprocessor post-close hooks at 1731640429290 (+6 ms)Closed at 1731640429291 (+1 ms) 2024-11-15T03:13:49,291 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,bbb,1731640423069.6130297c45bd23340b368de2cc708c90. 2024-11-15T03:13:49,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46785 is added to blk_1073741869_1045 (size=8195) 2024-11-15T03:13:49,291 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b1704275babe02e36b29cfaadf5b81af, disabling compactions & flushes 2024-11-15T03:13:49,291 INFO [RS_CLOSE_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.66 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/hbase/meta/1588230740/.tmp/info/a141ab00a364450599fca9e117c8a708 2024-11-15T03:13:49,292 INFO [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,jjj,1731640423069.b1704275babe02e36b29cfaadf5b81af. 2024-11-15T03:13:49,292 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1731640423069.b1704275babe02e36b29cfaadf5b81af. 2024-11-15T03:13:49,292 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1731640423069.b1704275babe02e36b29cfaadf5b81af. after waiting 0 ms 2024-11-15T03:13:49,292 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1731640423069.b1704275babe02e36b29cfaadf5b81af. 2024-11-15T03:13:49,294 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/b96380a8d7110dffa8cd7844252025dd/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T03:13:49,295 INFO [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ooo,1731640423069.b96380a8d7110dffa8cd7844252025dd. 2024-11-15T03:13:49,295 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b96380a8d7110dffa8cd7844252025dd: Waiting for close lock at 1731640429287Running coprocessor pre-close hooks at 1731640429287Disabling compacts and flushes for region at 1731640429287Disabling writes for close at 1731640429288 (+1 ms)Writing region close event to WAL at 1731640429288Running coprocessor post-close hooks at 1731640429295 (+7 ms)Closed at 1731640429295 2024-11-15T03:13:49,295 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ooo,1731640423069.b96380a8d7110dffa8cd7844252025dd. 2024-11-15T03:13:49,295 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 23f804d67a47ba185ffd6f6bc7b75464, disabling compactions & flushes 2024-11-15T03:13:49,295 INFO [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,qqq,1731640423069.23f804d67a47ba185ffd6f6bc7b75464. 2024-11-15T03:13:49,296 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1731640423069.23f804d67a47ba185ffd6f6bc7b75464. 2024-11-15T03:13:49,296 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1731640423069.23f804d67a47ba185ffd6f6bc7b75464. after waiting 0 ms 2024-11-15T03:13:49,296 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1731640423069.23f804d67a47ba185ffd6f6bc7b75464. 2024-11-15T03:13:49,298 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/b1704275babe02e36b29cfaadf5b81af/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T03:13:49,299 INFO [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,jjj,1731640423069.b1704275babe02e36b29cfaadf5b81af. 2024-11-15T03:13:49,299 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b1704275babe02e36b29cfaadf5b81af: Waiting for close lock at 1731640429291Running coprocessor pre-close hooks at 1731640429291Disabling compacts and flushes for region at 1731640429291Disabling writes for close at 1731640429292 (+1 ms)Writing region close event to WAL at 1731640429292Running coprocessor post-close hooks at 1731640429299 (+7 ms)Closed at 1731640429299 2024-11-15T03:13:49,299 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,jjj,1731640423069.b1704275babe02e36b29cfaadf5b81af. 2024-11-15T03:13:49,300 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 48ff514f5498660a4e8fc753340bb847, disabling compactions & flushes 2024-11-15T03:13:49,300 INFO [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,hhh,1731640423069.48ff514f5498660a4e8fc753340bb847. 2024-11-15T03:13:49,300 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1731640423069.48ff514f5498660a4e8fc753340bb847. 2024-11-15T03:13:49,300 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1731640423069.48ff514f5498660a4e8fc753340bb847. after waiting 0 ms 2024-11-15T03:13:49,300 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1731640423069.48ff514f5498660a4e8fc753340bb847. 2024-11-15T03:13:49,301 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/23f804d67a47ba185ffd6f6bc7b75464/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T03:13:49,302 DEBUG [RS_CLOSE_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/hbase/meta/1588230740/.tmp/info/a141ab00a364450599fca9e117c8a708 as hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/hbase/meta/1588230740/info/a141ab00a364450599fca9e117c8a708 2024-11-15T03:13:49,302 INFO [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,qqq,1731640423069.23f804d67a47ba185ffd6f6bc7b75464. 2024-11-15T03:13:49,302 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 23f804d67a47ba185ffd6f6bc7b75464: Waiting for close lock at 1731640429295Running coprocessor pre-close hooks at 1731640429295Disabling compacts and flushes for region at 1731640429295Disabling writes for close at 1731640429296 (+1 ms)Writing region close event to WAL at 1731640429296Running coprocessor post-close hooks at 1731640429302 (+6 ms)Closed at 1731640429302 2024-11-15T03:13:49,302 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,qqq,1731640423069.23f804d67a47ba185ffd6f6bc7b75464. 2024-11-15T03:13:49,303 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 860df8edb65f2fdd3c8bc00f89f94ba0, disabling compactions & flushes 2024-11-15T03:13:49,303 INFO [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,lll,1731640423069.860df8edb65f2fdd3c8bc00f89f94ba0. 2024-11-15T03:13:49,303 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1731640423069.860df8edb65f2fdd3c8bc00f89f94ba0. 2024-11-15T03:13:49,303 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,lll,1731640423069.860df8edb65f2fdd3c8bc00f89f94ba0. after waiting 0 ms 2024-11-15T03:13:49,303 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,lll,1731640423069.860df8edb65f2fdd3c8bc00f89f94ba0. 2024-11-15T03:13:49,305 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/48ff514f5498660a4e8fc753340bb847/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T03:13:49,306 INFO [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,hhh,1731640423069.48ff514f5498660a4e8fc753340bb847. 2024-11-15T03:13:49,306 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 48ff514f5498660a4e8fc753340bb847: Waiting for close lock at 1731640429300Running coprocessor pre-close hooks at 1731640429300Disabling compacts and flushes for region at 1731640429300Disabling writes for close at 1731640429300Writing region close event to WAL at 1731640429300Running coprocessor post-close hooks at 1731640429306 (+6 ms)Closed at 1731640429306 2024-11-15T03:13:49,306 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,hhh,1731640423069.48ff514f5498660a4e8fc753340bb847. 2024-11-15T03:13:49,307 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 155ef591549501ad2da24d2c559fb7f9, disabling compactions & flushes 2024-11-15T03:13:49,307 INFO [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,www,1731640423069.155ef591549501ad2da24d2c559fb7f9. 2024-11-15T03:13:49,307 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1731640423069.155ef591549501ad2da24d2c559fb7f9. 2024-11-15T03:13:49,307 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,www,1731640423069.155ef591549501ad2da24d2c559fb7f9. after waiting 0 ms 2024-11-15T03:13:49,307 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,www,1731640423069.155ef591549501ad2da24d2c559fb7f9. 2024-11-15T03:13:49,308 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/860df8edb65f2fdd3c8bc00f89f94ba0/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T03:13:49,309 INFO [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,lll,1731640423069.860df8edb65f2fdd3c8bc00f89f94ba0. 2024-11-15T03:13:49,309 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 860df8edb65f2fdd3c8bc00f89f94ba0: Waiting for close lock at 1731640429303Running coprocessor pre-close hooks at 1731640429303Disabling compacts and flushes for region at 1731640429303Disabling writes for close at 1731640429303Writing region close event to WAL at 1731640429303Running coprocessor post-close hooks at 1731640429309 (+6 ms)Closed at 1731640429309 2024-11-15T03:13:49,310 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,lll,1731640423069.860df8edb65f2fdd3c8bc00f89f94ba0. 2024-11-15T03:13:49,310 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e0f9e1e28ca8906f0f82f959a60c55ce, disabling compactions & flushes 2024-11-15T03:13:49,310 INFO [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,eee,1731640423069.e0f9e1e28ca8906f0f82f959a60c55ce. 2024-11-15T03:13:49,310 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1731640423069.e0f9e1e28ca8906f0f82f959a60c55ce. 2024-11-15T03:13:49,310 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,eee,1731640423069.e0f9e1e28ca8906f0f82f959a60c55ce. after waiting 0 ms 2024-11-15T03:13:49,310 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,eee,1731640423069.e0f9e1e28ca8906f0f82f959a60c55ce. 2024-11-15T03:13:49,312 INFO [RS_CLOSE_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/hbase/meta/1588230740/info/a141ab00a364450599fca9e117c8a708, entries=24, sequenceid=75, filesize=8.0 K 2024-11-15T03:13:49,313 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/155ef591549501ad2da24d2c559fb7f9/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T03:13:49,314 INFO [RS_CLOSE_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.66 KB/2720, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 69ms, sequenceid=75, compaction requested=false 2024-11-15T03:13:49,314 INFO [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,www,1731640423069.155ef591549501ad2da24d2c559fb7f9. 2024-11-15T03:13:49,314 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 155ef591549501ad2da24d2c559fb7f9: Waiting for close lock at 1731640429307Running coprocessor pre-close hooks at 1731640429307Disabling compacts and flushes for region at 1731640429307Disabling writes for close at 1731640429307Writing region close event to WAL at 1731640429307Running coprocessor post-close hooks at 1731640429314 (+7 ms)Closed at 1731640429314 2024-11-15T03:13:49,314 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,www,1731640423069.155ef591549501ad2da24d2c559fb7f9. 2024-11-15T03:13:49,314 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5971f307349447e7f24b0dd05b96cffc, disabling compactions & flushes 2024-11-15T03:13:49,314 INFO [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ccc,1731640423069.5971f307349447e7f24b0dd05b96cffc. 2024-11-15T03:13:49,314 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1731640423069.5971f307349447e7f24b0dd05b96cffc. 2024-11-15T03:13:49,314 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1731640423069.5971f307349447e7f24b0dd05b96cffc. after waiting 0 ms 2024-11-15T03:13:49,314 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1731640423069.5971f307349447e7f24b0dd05b96cffc. 2024-11-15T03:13:49,316 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/e0f9e1e28ca8906f0f82f959a60c55ce/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T03:13:49,317 INFO [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,eee,1731640423069.e0f9e1e28ca8906f0f82f959a60c55ce. 2024-11-15T03:13:49,317 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e0f9e1e28ca8906f0f82f959a60c55ce: Waiting for close lock at 1731640429310Running coprocessor pre-close hooks at 1731640429310Disabling compacts and flushes for region at 1731640429310Disabling writes for close at 1731640429310Writing region close event to WAL at 1731640429312 (+2 ms)Running coprocessor post-close hooks at 1731640429317 (+5 ms)Closed at 1731640429317 2024-11-15T03:13:49,317 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,eee,1731640423069.e0f9e1e28ca8906f0f82f959a60c55ce. 2024-11-15T03:13:49,318 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f5219cfe492c74eb8312930d328964b6, disabling compactions & flushes 2024-11-15T03:13:49,318 INFO [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,iii,1731640423069.f5219cfe492c74eb8312930d328964b6. 2024-11-15T03:13:49,318 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1731640423069.f5219cfe492c74eb8312930d328964b6. 2024-11-15T03:13:49,318 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,iii,1731640423069.f5219cfe492c74eb8312930d328964b6. after waiting 0 ms 2024-11-15T03:13:49,318 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,iii,1731640423069.f5219cfe492c74eb8312930d328964b6. 2024-11-15T03:13:49,321 DEBUG [RS_CLOSE_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/hbase/meta/1588230740/recovered.edits/78.seqid, newMaxSeqId=78, maxSeqId=64 2024-11-15T03:13:49,321 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/5971f307349447e7f24b0dd05b96cffc/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T03:13:49,322 INFO [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ccc,1731640423069.5971f307349447e7f24b0dd05b96cffc. 2024-11-15T03:13:49,322 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5971f307349447e7f24b0dd05b96cffc: Waiting for close lock at 1731640429314Running coprocessor pre-close hooks at 1731640429314Disabling compacts and flushes for region at 1731640429314Disabling writes for close at 1731640429314Writing region close event to WAL at 1731640429315 (+1 ms)Running coprocessor post-close hooks at 1731640429322 (+7 ms)Closed at 1731640429322 2024-11-15T03:13:49,322 DEBUG [RS_CLOSE_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-15T03:13:49,322 INFO [RS_CLOSE_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-15T03:13:49,322 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ccc,1731640423069.5971f307349447e7f24b0dd05b96cffc. 2024-11-15T03:13:49,322 DEBUG [RS_CLOSE_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731640429244Running coprocessor pre-close hooks at 1731640429244Disabling compacts and flushes for region at 1731640429244Disabling writes for close at 1731640429244Obtaining lock to block concurrent updates at 1731640429244Preparing flush snapshotting stores in 1588230740 at 1731640429244Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=2720, getHeapSize=5504, getOffHeapSize=0, getCellsCount=24 at 1731640429250 (+6 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731640429251 (+1 ms)Flushing 1588230740/info: creating writer at 1731640429252 (+1 ms)Flushing 1588230740/info: appending metadata at 1731640429264 (+12 ms)Flushing 1588230740/info: closing flushed file at 1731640429264Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@27b45591: reopening flushed file at 1731640429300 (+36 ms)Finished flush of dataSize ~2.66 KB/2720, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 69ms, sequenceid=75, compaction requested=false at 1731640429314 (+14 ms)Writing region close event to WAL at 1731640429316 (+2 ms)Running coprocessor post-close hooks at 1731640429322 (+6 ms)Closed at 1731640429322 2024-11-15T03:13:49,322 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing eaf85cc654192e46d3ea0904493eea12, disabling compactions & flushes 2024-11-15T03:13:49,322 DEBUG [RS_CLOSE_META-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-15T03:13:49,322 INFO [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,mmm,1731640423069.eaf85cc654192e46d3ea0904493eea12. 2024-11-15T03:13:49,322 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1731640423069.eaf85cc654192e46d3ea0904493eea12. 2024-11-15T03:13:49,322 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1731640423069.eaf85cc654192e46d3ea0904493eea12. after waiting 0 ms 2024-11-15T03:13:49,322 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1731640423069.eaf85cc654192e46d3ea0904493eea12. 2024-11-15T03:13:49,324 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/f5219cfe492c74eb8312930d328964b6/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T03:13:49,325 INFO [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,iii,1731640423069.f5219cfe492c74eb8312930d328964b6. 2024-11-15T03:13:49,325 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f5219cfe492c74eb8312930d328964b6: Waiting for close lock at 1731640429318Running coprocessor pre-close hooks at 1731640429318Disabling compacts and flushes for region at 1731640429318Disabling writes for close at 1731640429318Writing region close event to WAL at 1731640429320 (+2 ms)Running coprocessor post-close hooks at 1731640429325 (+5 ms)Closed at 1731640429325 2024-11-15T03:13:49,325 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,iii,1731640423069.f5219cfe492c74eb8312930d328964b6. 2024-11-15T03:13:49,325 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing dc93af816bdf94cf920a5adc31356783, disabling compactions & flushes 2024-11-15T03:13:49,325 INFO [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,vvv,1731640423069.dc93af816bdf94cf920a5adc31356783. 2024-11-15T03:13:49,325 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1731640423069.dc93af816bdf94cf920a5adc31356783. 2024-11-15T03:13:49,325 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1731640423069.dc93af816bdf94cf920a5adc31356783. after waiting 0 ms 2024-11-15T03:13:49,325 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1731640423069.dc93af816bdf94cf920a5adc31356783. 2024-11-15T03:13:49,327 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/eaf85cc654192e46d3ea0904493eea12/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T03:13:49,327 INFO [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,mmm,1731640423069.eaf85cc654192e46d3ea0904493eea12. 2024-11-15T03:13:49,327 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for eaf85cc654192e46d3ea0904493eea12: Waiting for close lock at 1731640429322Running coprocessor pre-close hooks at 1731640429322Disabling compacts and flushes for region at 1731640429322Disabling writes for close at 1731640429322Writing region close event to WAL at 1731640429323 (+1 ms)Running coprocessor post-close hooks at 1731640429327 (+4 ms)Closed at 1731640429327 2024-11-15T03:13:49,328 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,mmm,1731640423069.eaf85cc654192e46d3ea0904493eea12. 2024-11-15T03:13:49,328 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c4b611da4a73d4cfc99379713486e21c, disabling compactions & flushes 2024-11-15T03:13:49,328 INFO [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,fff,1731640423069.c4b611da4a73d4cfc99379713486e21c. 2024-11-15T03:13:49,328 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1731640423069.c4b611da4a73d4cfc99379713486e21c. 2024-11-15T03:13:49,328 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,fff,1731640423069.c4b611da4a73d4cfc99379713486e21c. after waiting 0 ms 2024-11-15T03:13:49,328 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,fff,1731640423069.c4b611da4a73d4cfc99379713486e21c. 2024-11-15T03:13:49,330 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/dc93af816bdf94cf920a5adc31356783/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-15T03:13:49,331 INFO [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,vvv,1731640423069.dc93af816bdf94cf920a5adc31356783. 2024-11-15T03:13:49,331 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for dc93af816bdf94cf920a5adc31356783: Waiting for close lock at 1731640429325Running coprocessor pre-close hooks at 1731640429325Disabling compacts and flushes for region at 1731640429325Disabling writes for close at 1731640429325Writing region close event to WAL at 1731640429326 (+1 ms)Running coprocessor post-close hooks at 1731640429331 (+5 ms)Closed at 1731640429331 2024-11-15T03:13:49,331 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,vvv,1731640423069.dc93af816bdf94cf920a5adc31356783. 2024-11-15T03:13:49,331 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1beb579fcf8e161a0b9c2051c671a8f2, disabling compactions & flushes 2024-11-15T03:13:49,331 INFO [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ddd,1731640423069.1beb579fcf8e161a0b9c2051c671a8f2. 2024-11-15T03:13:49,331 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1731640423069.1beb579fcf8e161a0b9c2051c671a8f2. 2024-11-15T03:13:49,331 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1731640423069.1beb579fcf8e161a0b9c2051c671a8f2. after waiting 0 ms 2024-11-15T03:13:49,332 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1731640423069.1beb579fcf8e161a0b9c2051c671a8f2. 2024-11-15T03:13:49,333 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/c4b611da4a73d4cfc99379713486e21c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T03:13:49,333 INFO [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,fff,1731640423069.c4b611da4a73d4cfc99379713486e21c. 2024-11-15T03:13:49,334 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c4b611da4a73d4cfc99379713486e21c: Waiting for close lock at 1731640429328Running coprocessor pre-close hooks at 1731640429328Disabling compacts and flushes for region at 1731640429328Disabling writes for close at 1731640429328Writing region close event to WAL at 1731640429328Running coprocessor post-close hooks at 1731640429333 (+5 ms)Closed at 1731640429333 2024-11-15T03:13:49,334 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,fff,1731640423069.c4b611da4a73d4cfc99379713486e21c. 2024-11-15T03:13:49,335 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/1beb579fcf8e161a0b9c2051c671a8f2/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T03:13:49,336 INFO [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ddd,1731640423069.1beb579fcf8e161a0b9c2051c671a8f2. 2024-11-15T03:13:49,336 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1beb579fcf8e161a0b9c2051c671a8f2: Waiting for close lock at 1731640429331Running coprocessor pre-close hooks at 1731640429331Disabling compacts and flushes for region at 1731640429331Disabling writes for close at 1731640429331Writing region close event to WAL at 1731640429332 (+1 ms)Running coprocessor post-close hooks at 1731640429336 (+4 ms)Closed at 1731640429336 2024-11-15T03:13:49,336 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ddd,1731640423069.1beb579fcf8e161a0b9c2051c671a8f2. 2024-11-15T03:13:49,337 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing dc930ecfca79abbf57ed28c20f79230f, disabling compactions & flushes 2024-11-15T03:13:49,337 INFO [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,rrr,1731640423069.dc930ecfca79abbf57ed28c20f79230f. 2024-11-15T03:13:49,337 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1731640423069.dc930ecfca79abbf57ed28c20f79230f. 2024-11-15T03:13:49,337 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1731640423069.dc930ecfca79abbf57ed28c20f79230f. after waiting 0 ms 2024-11-15T03:13:49,337 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1731640423069.dc930ecfca79abbf57ed28c20f79230f. 2024-11-15T03:13:49,341 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/dc930ecfca79abbf57ed28c20f79230f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T03:13:49,342 INFO [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,rrr,1731640423069.dc930ecfca79abbf57ed28c20f79230f. 2024-11-15T03:13:49,342 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for dc930ecfca79abbf57ed28c20f79230f: Waiting for close lock at 1731640429337Running coprocessor pre-close hooks at 1731640429337Disabling compacts and flushes for region at 1731640429337Disabling writes for close at 1731640429337Writing region close event to WAL at 1731640429337Running coprocessor post-close hooks at 1731640429342 (+5 ms)Closed at 1731640429342 2024-11-15T03:13:49,342 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,rrr,1731640423069.dc930ecfca79abbf57ed28c20f79230f. 2024-11-15T03:13:49,342 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4ef03d91ad777a77cf66f510e28419ba, disabling compactions & flushes 2024-11-15T03:13:49,342 INFO [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,sss,1731640423069.4ef03d91ad777a77cf66f510e28419ba. 2024-11-15T03:13:49,342 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1731640423069.4ef03d91ad777a77cf66f510e28419ba. 2024-11-15T03:13:49,342 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,sss,1731640423069.4ef03d91ad777a77cf66f510e28419ba. after waiting 0 ms 2024-11-15T03:13:49,342 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,sss,1731640423069.4ef03d91ad777a77cf66f510e28419ba. 2024-11-15T03:13:49,347 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/4ef03d91ad777a77cf66f510e28419ba/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T03:13:49,347 INFO [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,sss,1731640423069.4ef03d91ad777a77cf66f510e28419ba. 2024-11-15T03:13:49,348 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4ef03d91ad777a77cf66f510e28419ba: Waiting for close lock at 1731640429342Running coprocessor pre-close hooks at 1731640429342Disabling compacts and flushes for region at 1731640429342Disabling writes for close at 1731640429342Writing region close event to WAL at 1731640429343 (+1 ms)Running coprocessor post-close hooks at 1731640429347 (+4 ms)Closed at 1731640429347 2024-11-15T03:13:49,348 DEBUG [RS_CLOSE_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,sss,1731640423069.4ef03d91ad777a77cf66f510e28419ba. 2024-11-15T03:13:49,409 WARN [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,uuu,1731640423069.1b8f1ea215e8c2f78186b80dafed1b28., aborting... java.io.IOException: Failed to report opened region to master: test,uuu,1731640423069.1b8f1ea215e8c2f78186b80dafed1b28. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T03:13:49,409 ERROR [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2440): ***** ABORTING region server 91f82191efb8,37523,1731640420593: Failed to open region test,uuu,1731640423069.1b8f1ea215e8c2f78186b80dafed1b28. and can not recover ***** java.io.IOException: Failed to report opened region to master: test,uuu,1731640423069.1b8f1ea215e8c2f78186b80dafed1b28. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T03:13:49,409 ERROR [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-15T03:13:49,410 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-15T03:13:49,411 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-15T03:13:49,415 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-15T03:13:49,416 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-15T03:13:49,425 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 509352768 }, "NonHeapMemoryUsage": { "committed": 164233216, "init": 7667712, "max": -1, "used": 161633856 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "91f82191efb8", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 38879, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 114, "ProcessCallTime_min": 0, "ProcessCallTime_max": 16, "ProcessCallTime_mean": 2, "ProcessCallTime_25th_percentile": 4, "ProcessCallTime_median": 8, "ProcessCallTime_75th_percentile": 12, "ProcessCallTime_90th_percentile": 14, "ProcessCallTime_95th_percentile": 15, "ProcessCallTime_98th_percentile": 15, "ProcessCallTime_99th_percentile": 15, "ProcessCallTime_99.9th_percentile": 15, "ProcessCallTime_TimeRangeCount_0-1": 114, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 114, "QueueCallTime_min": 0, "QueueCallTime_max": 12, "QueueCallTime_mean": 1, "QueueCallTime_25th_percentile": 3, "QueueCallTime_median": 6, "QueueCallTime_75th_percentile": 9, "QueueCallTime_90th_percentile": 10, "QueueCallTime_95th_percentile": 11, "QueueCallTime_98th_percentile": 11, "QueueCallTime_99th_percentile": 11, "QueueCallTime_99.9th_percentile": 11, "QueueCallTime_TimeRangeCount_0-1": 114, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 114, "TotalCallTime_min": 0, "TotalCallTime_max": 17, "TotalCallTime_mean": 3, "TotalCallTime_25th_percentile": 4, "TotalCallTime_median": 8, "TotalCallTime_75th_percentile": 12, "TotalCallTime_90th_percentile": 15, "TotalCallTime_95th_percentile": 16, "TotalCallTime_98th_percentile": 16, "TotalCallTime_99th_percentile": 16, "TotalCallTime_99.9th_percentile": 16, "TotalCallTime_TimeRangeCount_0-1": 114, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 114, "ResponseSize_min": 0, "ResponseSize_max": 22358, "ResponseSize_mean": 599, "ResponseSize_25th_percentile": 5589, "ResponseSize_median": 11179, "ResponseSize_75th_percentile": 16768, "ResponseSize_90th_percentile": 20122, "ResponseSize_95th_percentile": 21240, "ResponseSize_98th_percentile": 21910, "ResponseSize_99th_percentile": 22134, "ResponseSize_99.9th_percentile": 22335, "ResponseSize_SizeRangeCount_0-10": 114, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 114, "RequestSize_min": 81, "RequestSize_max": 8095, "RequestSize_mean": 308, "RequestSize_25th_percentile": 2084, "RequestSize_median": 4088, "RequestSize_75th_percentile": 6091, "RequestSize_90th_percentile": 7293, "RequestSize_95th_percentile": 7694, "RequestSize_98th_percentile": 7934, "RequestSize_99th_percentile": 8014, "RequestSize_99.9th_percentile": 8086, "RequestSize_SizeRangeCount_0-10": 114, "sentBytes": 54860 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "91f82191efb8", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:64070", "tag.serverName": "91f82191efb8,37523,1731640420593", "tag.clusterId": "451b97df-43ba-4345-aef9-5c28b9dc1656", "tag.Context": "regionserver", "tag.Hostname": "91f82191efb8", "regionCount": 10, "storeCount": 9, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 2304, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1731640420593, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 3337, "localBytesRead": 3337, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 1.8, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 7466, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 17, "totalRowActionRequestCount": 9, "readRequestCount": 9, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 9, "rpcScanRequestCount": 9, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 8, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 9, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "ScanTime_TimeRangeCount_0-1": 9, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 8, "Put_min": 1, "Put_max": 3, "Put_mean": 1, "Put_25th_percentile": 1, "Put_median": 2, "Put_75th_percentile": 2, "Put_90th_percentile": 2, "Put_95th_percentile": 2, "Put_98th_percentile": 2, "Put_99th_percentile": 2, "Put_99.9th_percentile": 2, "Put_TimeRangeCount_0-1": 8, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 46699, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 30111, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 1, "FlushOutputSize_min": 46699, "FlushOutputSize_max": 46699, "FlushOutputSize_mean": 46699, "FlushOutputSize_25th_percentile": 46699, "FlushOutputSize_median": 46699, "FlushOutputSize_75th_percentile": 46699, "FlushOutputSize_90th_percentile": 46699, "FlushOutputSize_95th_percentile": 46699, "FlushOutputSize_98th_percentile": 46699, "FlushOutputSize_99th_percentile": 46699, "FlushOutputSize_99.9th_percentile": 46699, "FlushOutputSize_SizeRangeCount_0-10": 1, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 1, "FlushMemstoreSize_min": 30111, "FlushMemstoreSize_max": 30111, "FlushMemstoreSize_mean": 30111, "FlushMemstoreSize_25th_percentile": 30111, "FlushMemstoreSize_median": 30111, "FlushMemstoreSize_75th_percentile": 30111, "FlushMemstoreSize_90th_percentile": 30111, "FlushMemstoreSize_95th_percentile": 30111, "FlushMemstoreSize_98th_percentile": 30111, "FlushMemstoreSize_99th_percentile": 30111, "FlushMemstoreSize_99.9th_percentile": 30111, "FlushMemstoreSize_SizeRangeCount_0-10": 1, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 1, "FlushTime_min": 245, "FlushTime_max": 245, "FlushTime_mean": 245, "FlushTime_25th_percentile": 245, "FlushTime_median": 245, "FlushTime_75th_percentile": 245, "FlushTime_90th_percentile": 245, "FlushTime_95th_percentile": 245, "FlushTime_98th_percentile": 245, "FlushTime_99th_percentile": 245, "FlushTime_99.9th_percentile": 245, "FlushTime_TimeRangeCount_0-1": 1, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 9, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "ScanSize_SizeRangeCount_0-10": 9, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-15T03:13:49,428 WARN [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2470): Unable to report fatal error to master org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=91f82191efb8:42051 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:344) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportRSFatalError(RegionServerStatusProtos.java:17290) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.abort(HRegionServer.java:2467) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abortRegionServer(SingleProcessHBaseCluster.java:192) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$2.run(SingleProcessHBaseCluster.java:185) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abort(SingleProcessHBaseCluster.java:182) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.handleException(AssignRegionHandler.java:173) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:106) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=91f82191efb8:42051 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 15 more Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 15 more 2024-11-15T03:13:49,428 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(132): Open test,ggg,1731640423069.d0eff1b91bc9b77579a5c44f1c985985. 2024-11-15T03:13:49,429 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7752): Opening region: {ENCODED => d0eff1b91bc9b77579a5c44f1c985985, NAME => 'test,ggg,1731640423069.d0eff1b91bc9b77579a5c44f1c985985.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-11-15T03:13:49,429 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d0eff1b91bc9b77579a5c44f1c985985 2024-11-15T03:13:49,429 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(898): Instantiated test,ggg,1731640423069.d0eff1b91bc9b77579a5c44f1c985985.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T03:13:49,429 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7794): checking encryption for d0eff1b91bc9b77579a5c44f1c985985 2024-11-15T03:13:49,429 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7797): checking classloading for d0eff1b91bc9b77579a5c44f1c985985 2024-11-15T03:13:49,431 INFO [StoreOpener-d0eff1b91bc9b77579a5c44f1c985985-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d0eff1b91bc9b77579a5c44f1c985985 2024-11-15T03:13:49,432 INFO [StoreOpener-d0eff1b91bc9b77579a5c44f1c985985-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d0eff1b91bc9b77579a5c44f1c985985 columnFamilyName family 2024-11-15T03:13:49,432 DEBUG [StoreOpener-d0eff1b91bc9b77579a5c44f1c985985-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T03:13:49,433 INFO [StoreOpener-d0eff1b91bc9b77579a5c44f1c985985-1 {}] regionserver.HStore(327): Store=d0eff1b91bc9b77579a5c44f1c985985/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T03:13:49,433 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1038): replaying wal for d0eff1b91bc9b77579a5c44f1c985985 2024-11-15T03:13:49,434 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/d0eff1b91bc9b77579a5c44f1c985985 2024-11-15T03:13:49,435 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/d0eff1b91bc9b77579a5c44f1c985985 2024-11-15T03:13:49,436 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1048): stopping wal replay for d0eff1b91bc9b77579a5c44f1c985985 2024-11-15T03:13:49,436 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1060): Cleaning up temporary data for d0eff1b91bc9b77579a5c44f1c985985 2024-11-15T03:13:49,437 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1093): writing seq id for d0eff1b91bc9b77579a5c44f1c985985 2024-11-15T03:13:49,438 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1114): Opened d0eff1b91bc9b77579a5c44f1c985985; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71113467, jitterRate=0.059673234820365906}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T03:13:49,439 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d0eff1b91bc9b77579a5c44f1c985985 2024-11-15T03:13:49,439 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1006): Region open journal for d0eff1b91bc9b77579a5c44f1c985985: Running coprocessor pre-open hook at 1731640429429Writing region info on filesystem at 1731640429429Initializing all the Stores at 1731640429430 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731640429430Cleaning up temporary data from old regions at 1731640429436 (+6 ms)Running coprocessor post-open hooks at 1731640429439 (+3 ms)Region opened successfully at 1731640429439 2024-11-15T03:13:49,440 WARN [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,ggg,1731640423069.d0eff1b91bc9b77579a5c44f1c985985., aborting... org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 91f82191efb8,37523,1731640420593 aborting at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkOpen(RSRpcServices.java:1463) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2235) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T03:13:49,440 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2435): Abort already in progress. Ignoring the current request with reason: Failed to open region test,ggg,1731640423069.d0eff1b91bc9b77579a5c44f1c985985. and can not recover 2024-11-15T03:13:49,440 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(132): Open test,nnn,1731640423069.a73d2d096e2f14a7e9365ac762927d55. 2024-11-15T03:13:49,441 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7752): Opening region: {ENCODED => a73d2d096e2f14a7e9365ac762927d55, NAME => 'test,nnn,1731640423069.a73d2d096e2f14a7e9365ac762927d55.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-11-15T03:13:49,441 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test a73d2d096e2f14a7e9365ac762927d55 2024-11-15T03:13:49,441 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(898): Instantiated test,nnn,1731640423069.a73d2d096e2f14a7e9365ac762927d55.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T03:13:49,441 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7794): checking encryption for a73d2d096e2f14a7e9365ac762927d55 2024-11-15T03:13:49,441 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7797): checking classloading for a73d2d096e2f14a7e9365ac762927d55 2024-11-15T03:13:49,443 INFO [StoreOpener-a73d2d096e2f14a7e9365ac762927d55-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region a73d2d096e2f14a7e9365ac762927d55 2024-11-15T03:13:49,444 INFO [StoreOpener-a73d2d096e2f14a7e9365ac762927d55-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a73d2d096e2f14a7e9365ac762927d55 columnFamilyName family 2024-11-15T03:13:49,444 DEBUG [StoreOpener-a73d2d096e2f14a7e9365ac762927d55-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T03:13:49,444 INFO [RS:2;91f82191efb8:41451 {}] regionserver.HRegionServer(976): stopping server 91f82191efb8,41451,1731640420638; all regions closed. 2024-11-15T03:13:49,444 INFO [RS:1;91f82191efb8:37523 {}] regionserver.HRegionServer(976): stopping server 91f82191efb8,37523,1731640420593; all regions closed. 2024-11-15T03:13:49,444 INFO [StoreOpener-a73d2d096e2f14a7e9365ac762927d55-1 {}] regionserver.HStore(327): Store=a73d2d096e2f14a7e9365ac762927d55/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T03:13:49,444 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1038): replaying wal for a73d2d096e2f14a7e9365ac762927d55 2024-11-15T03:13:49,446 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/a73d2d096e2f14a7e9365ac762927d55 2024-11-15T03:13:49,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46785 is added to blk_1073741833_1009 (size=7064) 2024-11-15T03:13:49,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38319 is added to blk_1073741833_1009 (size=7064) 2024-11-15T03:13:49,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36559 is added to blk_1073741833_1009 (size=7064) 2024-11-15T03:13:49,448 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/data/default/test/a73d2d096e2f14a7e9365ac762927d55 2024-11-15T03:13:49,449 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1048): stopping wal replay for a73d2d096e2f14a7e9365ac762927d55 2024-11-15T03:13:49,449 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1060): Cleaning up temporary data for a73d2d096e2f14a7e9365ac762927d55 2024-11-15T03:13:49,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46785 is added to blk_1073741865_1041 (size=4204) 2024-11-15T03:13:49,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38319 is added to blk_1073741865_1041 (size=4204) 2024-11-15T03:13:49,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36559 is added to blk_1073741865_1041 (size=4204) 2024-11-15T03:13:49,452 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1093): writing seq id for a73d2d096e2f14a7e9365ac762927d55 2024-11-15T03:13:49,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36559 is added to blk_1073741834_1010 (size=5999) 2024-11-15T03:13:49,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46785 is added to blk_1073741834_1010 (size=5999) 2024-11-15T03:13:49,455 DEBUG [RS:2;91f82191efb8:41451 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/oldWALs 2024-11-15T03:13:49,455 INFO [RS:2;91f82191efb8:41451 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 91f82191efb8%2C41451%2C1731640420638:(num 1731640422194) 2024-11-15T03:13:49,455 DEBUG [RS:2;91f82191efb8:41451 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T03:13:49,455 INFO [RS:2;91f82191efb8:41451 {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T03:13:49,455 INFO [RS:2;91f82191efb8:41451 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-15T03:13:49,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38319 is added to blk_1073741834_1010 (size=5999) 2024-11-15T03:13:49,456 DEBUG [RS:1;91f82191efb8:37523 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T03:13:49,456 INFO [RS:1;91f82191efb8:37523 {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T03:13:49,456 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1114): Opened a73d2d096e2f14a7e9365ac762927d55; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72445157, jitterRate=0.07951696217060089}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T03:13:49,456 INFO [RS:2;91f82191efb8:41451 {}] hbase.ChoreService(370): Chore service for: regionserver/91f82191efb8:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-15T03:13:49,456 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a73d2d096e2f14a7e9365ac762927d55 2024-11-15T03:13:49,456 INFO [RS:1;91f82191efb8:37523 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-15T03:13:49,456 INFO [RS:2;91f82191efb8:41451 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-15T03:13:49,456 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1006): Region open journal for a73d2d096e2f14a7e9365ac762927d55: Running coprocessor pre-open hook at 1731640429441Writing region info on filesystem at 1731640429441Initializing all the Stores at 1731640429442 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731640429442Cleaning up temporary data from old regions at 1731640429449 (+7 ms)Running coprocessor post-open hooks at 1731640429456 (+7 ms)Region opened successfully at 1731640429456 2024-11-15T03:13:49,456 INFO [RS:2;91f82191efb8:41451 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-15T03:13:49,456 INFO [regionserver/91f82191efb8:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-15T03:13:49,456 INFO [RS:2;91f82191efb8:41451 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-15T03:13:49,456 INFO [RS:1;91f82191efb8:37523 {}] hbase.ChoreService(370): Chore service for: regionserver/91f82191efb8:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-15T03:13:49,456 INFO [RS:2;91f82191efb8:41451 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-15T03:13:49,456 INFO [RS:1;91f82191efb8:37523 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-15T03:13:49,456 INFO [regionserver/91f82191efb8:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-15T03:13:49,456 INFO [RS:1;91f82191efb8:37523 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37523 2024-11-15T03:13:49,456 INFO [RS:2;91f82191efb8:41451 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41451 2024-11-15T03:13:49,457 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1722): Closing a73d2d096e2f14a7e9365ac762927d55, disabling compactions & flushes 2024-11-15T03:13:49,457 INFO [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1755): Closing region test,nnn,1731640423069.a73d2d096e2f14a7e9365ac762927d55. 2024-11-15T03:13:49,458 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1731640423069.a73d2d096e2f14a7e9365ac762927d55. 2024-11-15T03:13:49,458 WARN [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1802): Interrupted while waiting for close lock on test,nnn,1731640423069.a73d2d096e2f14a7e9365ac762927d55. java.lang.InterruptedException: null at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireNanos(AbstractQueuedSynchronizer.java:993) ~[?:?] at java.util.concurrent.locks.ReentrantReadWriteLock$WriteLock.tryLock(ReentrantReadWriteLock.java:1131) ~[?:?] at org.apache.hadoop.hbase.regionserver.HRegion.doClose(HRegion.java:1794) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.close(HRegion.java:1672) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.close(HRegion.java:1627) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.close(HRegion.java:1610) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7817) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegionFromTableDir(HRegion.java:7754) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7729) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7688) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7644) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:143) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T03:13:49,458 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1676): Region close journal for a73d2d096e2f14a7e9365ac762927d55: Waiting for close lock at 1731640429457Running coprocessor pre-close hooks at 1731640429457Disabling compacts and flushes for region at 1731640429457 2024-11-15T03:13:49,458 WARN [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7819): Open region: {ENCODED => a73d2d096e2f14a7e9365ac762927d55, NAME => 'test,nnn,1731640423069.a73d2d096e2f14a7e9365ac762927d55.', STARTKEY => 'nnn', ENDKEY => 'ooo'} failed. Try close region but got exception java.io.InterruptedIOException: Interrupted while waiting for close lock on test,nnn,1731640423069.a73d2d096e2f14a7e9365ac762927d55. at org.apache.hadoop.hbase.regionserver.HRegion.doClose(HRegion.java:1803) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.close(HRegion.java:1672) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.close(HRegion.java:1627) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.close(HRegion.java:1610) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7817) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegionFromTableDir(HRegion.java:7754) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7729) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7688) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7644) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:143) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireNanos(AbstractQueuedSynchronizer.java:993) ~[?:?] at java.util.concurrent.locks.ReentrantReadWriteLock$WriteLock.tryLock(ReentrantReadWriteLock.java:1131) ~[?:?] at org.apache.hadoop.hbase.regionserver.HRegion.doClose(HRegion.java:1794) ~[classes/:?] ... 13 more 2024-11-15T03:13:49,459 WARN [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(84): Failed to open region test,nnn,1731640423069.a73d2d096e2f14a7e9365ac762927d55., will report to master java.io.IOException: Cannot append; log is closed, regionName = test,nnn,1731640423069.a73d2d096e2f14a7e9365ac762927d55. at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.stampSequenceIdAndPublishToRingBuffer(AbstractFSWAL.java:1393) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.append(AbstractFSWAL.java:1920) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$appendMarker$14(AbstractFSWAL.java:1455) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendMarker(AbstractFSWAL.java:1455) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.WALUtil.doFullMarkerAppendTransaction(WALUtil.java:168) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.WALUtil.writeMarker(WALUtil.java:148) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.WALUtil.writeRegionEventMarker(WALUtil.java:112) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.writeRegionOpenMarker(HRegion.java:1269) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7808) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegionFromTableDir(HRegion.java:7754) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7729) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7688) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7644) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:143) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T03:13:49,459 WARN [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,nnn,1731640423069.a73d2d096e2f14a7e9365ac762927d55., aborting... java.io.IOException: Failed to report failed open to master: test,nnn,1731640423069.a73d2d096e2f14a7e9365ac762927d55. at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.cleanUpAndReportFailure(AssignRegionHandler.java:93) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:145) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T03:13:49,459 DEBUG [RS_OPEN_REGION-regionserver/91f82191efb8:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2435): Abort already in progress. Ignoring the current request with reason: Failed to open region test,nnn,1731640423069.a73d2d096e2f14a7e9365ac762927d55. and can not recover 2024-11-15T03:13:49,467 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42051-0x1013c7bdd830000, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-15T03:13:49,467 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37523-0x1013c7bdd830002, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/91f82191efb8,37523,1731640420593 2024-11-15T03:13:49,467 INFO [RS:1;91f82191efb8:37523 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-15T03:13:49,477 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41451-0x1013c7bdd830003, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/91f82191efb8,41451,1731640420638 2024-11-15T03:13:49,477 INFO [RS:2;91f82191efb8:41451 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-15T03:13:49,488 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [91f82191efb8,41451,1731640420638] 2024-11-15T03:13:49,509 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/91f82191efb8,41451,1731640420638 already deleted, retry=false 2024-11-15T03:13:49,509 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 91f82191efb8,41451,1731640420638 expired; onlineServers=1 2024-11-15T03:13:49,509 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [91f82191efb8,37523,1731640420593] 2024-11-15T03:13:49,519 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/91f82191efb8,37523,1731640420593 already deleted, retry=false 2024-11-15T03:13:49,519 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 91f82191efb8,37523,1731640420593 expired; onlineServers=0 2024-11-15T03:13:49,519 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '91f82191efb8,42051,1731640419585' ***** 2024-11-15T03:13:49,520 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-15T03:13:49,520 INFO [M:0;91f82191efb8:42051 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-15T03:13:49,520 INFO [M:0;91f82191efb8:42051 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-15T03:13:49,520 DEBUG [M:0;91f82191efb8:42051 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-15T03:13:49,520 DEBUG [M:0;91f82191efb8:42051 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-15T03:13:49,520 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-15T03:13:49,520 DEBUG [master/91f82191efb8:0:becomeActiveMaster-HFileCleaner.small.0-1731640421833 {}] cleaner.HFileCleaner(306): Exit Thread[master/91f82191efb8:0:becomeActiveMaster-HFileCleaner.small.0-1731640421833,5,FailOnTimeoutGroup] 2024-11-15T03:13:49,520 DEBUG [master/91f82191efb8:0:becomeActiveMaster-HFileCleaner.large.0-1731640421829 {}] cleaner.HFileCleaner(306): Exit Thread[master/91f82191efb8:0:becomeActiveMaster-HFileCleaner.large.0-1731640421829,5,FailOnTimeoutGroup] 2024-11-15T03:13:49,520 INFO [M:0;91f82191efb8:42051 {}] hbase.ChoreService(370): Chore service for: master/91f82191efb8:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-15T03:13:49,521 INFO [M:0;91f82191efb8:42051 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-15T03:13:49,521 DEBUG [M:0;91f82191efb8:42051 {}] master.HMaster(1795): Stopping service threads 2024-11-15T03:13:49,521 INFO [M:0;91f82191efb8:42051 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-15T03:13:49,521 INFO [M:0;91f82191efb8:42051 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T03:13:49,521 INFO [M:0;91f82191efb8:42051 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T03:13:49,521 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-15T03:13:49,530 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42051-0x1013c7bdd830000, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-15T03:13:49,530 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42051-0x1013c7bdd830000, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T03:13:49,530 DEBUG [M:0;91f82191efb8:42051 {}] zookeeper.ZKUtil(347): master:42051-0x1013c7bdd830000, quorum=127.0.0.1:64070, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-15T03:13:49,530 WARN [M:0;91f82191efb8:42051 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-15T03:13:49,532 INFO [M:0;91f82191efb8:42051 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/.lastflushedseqids 2024-11-15T03:13:49,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36559 is added to blk_1073741870_1046 (size=1) 2024-11-15T03:13:49,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46785 is added to blk_1073741870_1046 (size=1) 2024-11-15T03:13:49,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38319 is added to blk_1073741870_1046 (size=1) 2024-11-15T03:13:49,543 INFO [M:0;91f82191efb8:42051 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T03:13:49,543 INFO [M:0;91f82191efb8:42051 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-15T03:13:49,544 DEBUG [M:0;91f82191efb8:42051 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-15T03:13:49,544 INFO [M:0;91f82191efb8:42051 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T03:13:49,544 DEBUG [M:0;91f82191efb8:42051 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T03:13:49,544 DEBUG [M:0;91f82191efb8:42051 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-15T03:13:49,544 DEBUG [M:0;91f82191efb8:42051 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T03:13:49,544 INFO [M:0;91f82191efb8:42051 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=259.86 KB heapSize=315.84 KB 2024-11-15T03:13:49,559 DEBUG [M:0;91f82191efb8:42051 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4629463cf71f4ad6b989a030f3c83479 is 82, key is hbase:meta,,1/info:regioninfo/1731640427639/Put/seqid=0 2024-11-15T03:13:49,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46785 is added to blk_1073741871_1047 (size=6071) 2024-11-15T03:13:49,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36559 is added to blk_1073741871_1047 (size=6071) 2024-11-15T03:13:49,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38319 is added to blk_1073741871_1047 (size=6071) 2024-11-15T03:13:49,567 INFO [M:0;91f82191efb8:42051 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=600 (bloomFilter=true), to=hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4629463cf71f4ad6b989a030f3c83479 2024-11-15T03:13:49,588 INFO [RS:1;91f82191efb8:37523 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-15T03:13:49,588 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37523-0x1013c7bdd830002, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T03:13:49,588 INFO [RS:1;91f82191efb8:37523 {}] regionserver.HRegionServer(1031): Exiting; stopping=91f82191efb8,37523,1731640420593; zookeeper connection closed. 2024-11-15T03:13:49,588 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37523-0x1013c7bdd830002, quorum=127.0.0.1:64070, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T03:13:49,588 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3bee696a {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3bee696a 2024-11-15T03:13:49,591 DEBUG [M:0;91f82191efb8:42051 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38383/user/jenkins/test-data/6baa8243-27fa-178e-e845-43a64c8fde46/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d0bb485219844c859f9aa97626e72717 is 2990, key is \x00\x00\x00\x00\x00\x00\x00