2024-11-27 13:40:33,006 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-27 13:40:33,024 main DEBUG Took 0.015482 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-27 13:40:33,024 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-27 13:40:33,025 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-27 13:40:33,026 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-27 13:40:33,027 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 13:40:33,034 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-27 13:40:33,046 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 13:40:33,047 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 13:40:33,048 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 13:40:33,049 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 13:40:33,049 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 13:40:33,050 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 13:40:33,051 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 13:40:33,051 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 13:40:33,052 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 13:40:33,052 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 13:40:33,053 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 13:40:33,053 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 13:40:33,054 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 13:40:33,054 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 13:40:33,055 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 13:40:33,055 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 13:40:33,056 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 13:40:33,056 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 13:40:33,056 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 13:40:33,057 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 13:40:33,057 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 13:40:33,057 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 13:40:33,058 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 13:40:33,058 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 13:40:33,059 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 13:40:33,059 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-27 13:40:33,060 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 13:40:33,062 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-27 13:40:33,064 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-27 13:40:33,064 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-27 13:40:33,066 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-27 13:40:33,066 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-27 13:40:33,074 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-27 13:40:33,077 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-27 13:40:33,079 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-27 13:40:33,079 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-27 13:40:33,080 main DEBUG createAppenders(={Console}) 2024-11-27 13:40:33,081 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 initialized 2024-11-27 13:40:33,081 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-27 13:40:33,081 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 OK. 2024-11-27 13:40:33,082 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-27 13:40:33,082 main DEBUG OutputStream closed 2024-11-27 13:40:33,083 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-27 13:40:33,083 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-27 13:40:33,083 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@c88a337 OK 2024-11-27 13:40:33,163 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-27 13:40:33,166 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-27 13:40:33,167 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-27 13:40:33,169 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-27 13:40:33,170 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-27 13:40:33,170 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-27 13:40:33,170 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-27 13:40:33,171 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-27 13:40:33,171 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-27 13:40:33,171 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-27 13:40:33,172 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-27 13:40:33,172 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-27 13:40:33,172 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-27 13:40:33,173 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-27 13:40:33,173 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-27 13:40:33,173 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-27 13:40:33,173 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-27 13:40:33,174 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-27 13:40:33,176 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-27 13:40:33,176 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@13330ac6) with optional ClassLoader: null 2024-11-27 13:40:33,177 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-27 13:40:33,177 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@13330ac6] started OK. 2024-11-27T13:40:33,195 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP timeout: 13 mins 2024-11-27 13:40:33,199 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-27 13:40:33,199 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-27T13:40:33,433 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8706bd9e-726a-c9fe-5644-8f636122acec 2024-11-27T13:40:33,451 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$HMasterForTest, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-27T13:40:33,472 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8706bd9e-726a-c9fe-5644-8f636122acec/cluster_192d029c-90d2-1a83-1d8c-c62dd1a1bf62, deleteOnExit=true 2024-11-27T13:40:33,473 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-27T13:40:33,475 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8706bd9e-726a-c9fe-5644-8f636122acec/test.cache.data in system properties and HBase conf 2024-11-27T13:40:33,476 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8706bd9e-726a-c9fe-5644-8f636122acec/hadoop.tmp.dir in system properties and HBase conf 2024-11-27T13:40:33,477 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8706bd9e-726a-c9fe-5644-8f636122acec/hadoop.log.dir in system properties and HBase conf 2024-11-27T13:40:33,478 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8706bd9e-726a-c9fe-5644-8f636122acec/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-27T13:40:33,479 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8706bd9e-726a-c9fe-5644-8f636122acec/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-27T13:40:33,479 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-27T13:40:33,589 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-27T13:40:33,676 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-27T13:40:33,679 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8706bd9e-726a-c9fe-5644-8f636122acec/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-27T13:40:33,680 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8706bd9e-726a-c9fe-5644-8f636122acec/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-27T13:40:33,680 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8706bd9e-726a-c9fe-5644-8f636122acec/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-27T13:40:33,681 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8706bd9e-726a-c9fe-5644-8f636122acec/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-27T13:40:33,681 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8706bd9e-726a-c9fe-5644-8f636122acec/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-27T13:40:33,682 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8706bd9e-726a-c9fe-5644-8f636122acec/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-27T13:40:33,682 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8706bd9e-726a-c9fe-5644-8f636122acec/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-27T13:40:33,683 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8706bd9e-726a-c9fe-5644-8f636122acec/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-27T13:40:33,683 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8706bd9e-726a-c9fe-5644-8f636122acec/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-27T13:40:33,683 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8706bd9e-726a-c9fe-5644-8f636122acec/nfs.dump.dir in system properties and HBase conf 2024-11-27T13:40:33,684 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8706bd9e-726a-c9fe-5644-8f636122acec/java.io.tmpdir in system properties and HBase conf 2024-11-27T13:40:33,684 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8706bd9e-726a-c9fe-5644-8f636122acec/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-27T13:40:33,684 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8706bd9e-726a-c9fe-5644-8f636122acec/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-27T13:40:33,685 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8706bd9e-726a-c9fe-5644-8f636122acec/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-27T13:40:34,773 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-27T13:40:34,840 INFO [Time-limited test {}] log.Log(170): Logging initialized @2516ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-27T13:40:34,906 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T13:40:34,967 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-27T13:40:34,991 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-27T13:40:34,991 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-27T13:40:34,992 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-27T13:40:35,006 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T13:40:35,009 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3912242{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8706bd9e-726a-c9fe-5644-8f636122acec/hadoop.log.dir/,AVAILABLE} 2024-11-27T13:40:35,010 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@595320c8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-27T13:40:35,205 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@64418798{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8706bd9e-726a-c9fe-5644-8f636122acec/java.io.tmpdir/jetty-localhost-46469-hadoop-hdfs-3_4_1-tests_jar-_-any-12961049004623665417/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-27T13:40:35,210 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@11744524{HTTP/1.1, (http/1.1)}{localhost:46469} 2024-11-27T13:40:35,211 INFO [Time-limited test {}] server.Server(415): Started @2887ms 2024-11-27T13:40:35,768 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T13:40:35,777 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-27T13:40:35,778 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-27T13:40:35,778 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-27T13:40:35,779 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-27T13:40:35,779 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3a300759{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8706bd9e-726a-c9fe-5644-8f636122acec/hadoop.log.dir/,AVAILABLE} 2024-11-27T13:40:35,780 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@474d518f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-27T13:40:35,892 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@79b30fb0{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8706bd9e-726a-c9fe-5644-8f636122acec/java.io.tmpdir/jetty-localhost-40989-hadoop-hdfs-3_4_1-tests_jar-_-any-16108478437376921155/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-27T13:40:35,893 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@548cd725{HTTP/1.1, (http/1.1)}{localhost:40989} 2024-11-27T13:40:35,893 INFO [Time-limited test {}] server.Server(415): Started @3570ms 2024-11-27T13:40:35,949 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-27T13:40:36,059 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T13:40:36,067 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-27T13:40:36,072 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-27T13:40:36,072 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-27T13:40:36,073 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-27T13:40:36,075 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@29652cf4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8706bd9e-726a-c9fe-5644-8f636122acec/hadoop.log.dir/,AVAILABLE} 2024-11-27T13:40:36,076 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1b0a32ba{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-27T13:40:36,184 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@34fcb585{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8706bd9e-726a-c9fe-5644-8f636122acec/java.io.tmpdir/jetty-localhost-43635-hadoop-hdfs-3_4_1-tests_jar-_-any-6076225514970161452/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-27T13:40:36,185 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@389bb6f7{HTTP/1.1, (http/1.1)}{localhost:43635} 2024-11-27T13:40:36,185 INFO [Time-limited test {}] server.Server(415): Started @3862ms 2024-11-27T13:40:36,187 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-27T13:40:36,225 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T13:40:36,230 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-27T13:40:36,232 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-27T13:40:36,233 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-27T13:40:36,233 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-27T13:40:36,234 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@290e2eec{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8706bd9e-726a-c9fe-5644-8f636122acec/hadoop.log.dir/,AVAILABLE} 2024-11-27T13:40:36,234 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@417efacb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-27T13:40:36,332 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3c1f701{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8706bd9e-726a-c9fe-5644-8f636122acec/java.io.tmpdir/jetty-localhost-45161-hadoop-hdfs-3_4_1-tests_jar-_-any-16439737693891641437/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-27T13:40:36,333 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3d599bb4{HTTP/1.1, (http/1.1)}{localhost:45161} 2024-11-27T13:40:36,334 INFO [Time-limited test {}] server.Server(415): Started @4010ms 2024-11-27T13:40:36,335 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-27T13:40:37,624 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8706bd9e-726a-c9fe-5644-8f636122acec/cluster_192d029c-90d2-1a83-1d8c-c62dd1a1bf62/data/data4/current/BP-1944179033-172.17.0.2-1732714834229/current, will proceed with Du for space computation calculation, 2024-11-27T13:40:37,624 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8706bd9e-726a-c9fe-5644-8f636122acec/cluster_192d029c-90d2-1a83-1d8c-c62dd1a1bf62/data/data3/current/BP-1944179033-172.17.0.2-1732714834229/current, will proceed with Du for space computation calculation, 2024-11-27T13:40:37,624 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8706bd9e-726a-c9fe-5644-8f636122acec/cluster_192d029c-90d2-1a83-1d8c-c62dd1a1bf62/data/data1/current/BP-1944179033-172.17.0.2-1732714834229/current, will proceed with Du for space computation calculation, 2024-11-27T13:40:37,624 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8706bd9e-726a-c9fe-5644-8f636122acec/cluster_192d029c-90d2-1a83-1d8c-c62dd1a1bf62/data/data2/current/BP-1944179033-172.17.0.2-1732714834229/current, will proceed with Du for space computation calculation, 2024-11-27T13:40:37,661 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-27T13:40:37,662 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-27T13:40:37,710 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2ca04a980ebb7b17 with lease ID 0xa73f8761e94e3fd0: Processing first storage report for DS-fac1a99f-a8dd-4363-b07e-bd13cdba036b from datanode DatanodeRegistration(127.0.0.1:38955, datanodeUuid=1e6b559b-6530-485d-8c8d-7b2b8f3a5bf6, infoPort=36243, infoSecurePort=0, ipcPort=45957, storageInfo=lv=-57;cid=testClusterID;nsid=127245085;c=1732714834229) 2024-11-27T13:40:37,712 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2ca04a980ebb7b17 with lease ID 0xa73f8761e94e3fd0: from storage DS-fac1a99f-a8dd-4363-b07e-bd13cdba036b node DatanodeRegistration(127.0.0.1:38955, datanodeUuid=1e6b559b-6530-485d-8c8d-7b2b8f3a5bf6, infoPort=36243, infoSecurePort=0, ipcPort=45957, storageInfo=lv=-57;cid=testClusterID;nsid=127245085;c=1732714834229), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-27T13:40:37,712 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa9bca0a878df4bf with lease ID 0xa73f8761e94e3fcf: Processing first storage report for DS-d60122c6-962a-4258-9fbb-6d2f24e1c6ac from datanode DatanodeRegistration(127.0.0.1:42991, datanodeUuid=fced76f7-d573-4c9e-914b-bbbe29602243, infoPort=34247, infoSecurePort=0, ipcPort=37963, storageInfo=lv=-57;cid=testClusterID;nsid=127245085;c=1732714834229) 2024-11-27T13:40:37,713 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa9bca0a878df4bf with lease ID 0xa73f8761e94e3fcf: from storage DS-d60122c6-962a-4258-9fbb-6d2f24e1c6ac node DatanodeRegistration(127.0.0.1:42991, datanodeUuid=fced76f7-d573-4c9e-914b-bbbe29602243, infoPort=34247, infoSecurePort=0, ipcPort=37963, storageInfo=lv=-57;cid=testClusterID;nsid=127245085;c=1732714834229), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-27T13:40:37,713 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2ca04a980ebb7b17 with lease ID 0xa73f8761e94e3fd0: Processing first storage report for DS-d5542cfc-7551-40a7-9fb5-1830b67c7811 from datanode DatanodeRegistration(127.0.0.1:38955, datanodeUuid=1e6b559b-6530-485d-8c8d-7b2b8f3a5bf6, infoPort=36243, infoSecurePort=0, ipcPort=45957, storageInfo=lv=-57;cid=testClusterID;nsid=127245085;c=1732714834229) 2024-11-27T13:40:37,713 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2ca04a980ebb7b17 with lease ID 0xa73f8761e94e3fd0: from storage DS-d5542cfc-7551-40a7-9fb5-1830b67c7811 node DatanodeRegistration(127.0.0.1:38955, datanodeUuid=1e6b559b-6530-485d-8c8d-7b2b8f3a5bf6, infoPort=36243, infoSecurePort=0, ipcPort=45957, storageInfo=lv=-57;cid=testClusterID;nsid=127245085;c=1732714834229), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-27T13:40:37,714 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa9bca0a878df4bf with lease ID 0xa73f8761e94e3fcf: Processing first storage report for DS-5d2570bf-b1b0-4cf0-a87a-b6122f73c0be from datanode DatanodeRegistration(127.0.0.1:42991, datanodeUuid=fced76f7-d573-4c9e-914b-bbbe29602243, infoPort=34247, infoSecurePort=0, ipcPort=37963, storageInfo=lv=-57;cid=testClusterID;nsid=127245085;c=1732714834229) 2024-11-27T13:40:37,714 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa9bca0a878df4bf with lease ID 0xa73f8761e94e3fcf: from storage DS-5d2570bf-b1b0-4cf0-a87a-b6122f73c0be node DatanodeRegistration(127.0.0.1:42991, datanodeUuid=fced76f7-d573-4c9e-914b-bbbe29602243, infoPort=34247, infoSecurePort=0, ipcPort=37963, storageInfo=lv=-57;cid=testClusterID;nsid=127245085;c=1732714834229), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-27T13:40:37,786 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8706bd9e-726a-c9fe-5644-8f636122acec/cluster_192d029c-90d2-1a83-1d8c-c62dd1a1bf62/data/data6/current/BP-1944179033-172.17.0.2-1732714834229/current, will proceed with Du for space computation calculation, 2024-11-27T13:40:37,786 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8706bd9e-726a-c9fe-5644-8f636122acec/cluster_192d029c-90d2-1a83-1d8c-c62dd1a1bf62/data/data5/current/BP-1944179033-172.17.0.2-1732714834229/current, will proceed with Du for space computation calculation, 2024-11-27T13:40:37,809 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-27T13:40:37,814 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd76af8ca9b8693d9 with lease ID 0xa73f8761e94e3fd1: Processing first storage report for DS-608d5eb3-8a33-4125-bd6d-2845739e66fc from datanode DatanodeRegistration(127.0.0.1:37007, datanodeUuid=ad7930af-671f-4232-96bc-4c380afba2bb, infoPort=36867, infoSecurePort=0, ipcPort=45355, storageInfo=lv=-57;cid=testClusterID;nsid=127245085;c=1732714834229) 2024-11-27T13:40:37,815 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd76af8ca9b8693d9 with lease ID 0xa73f8761e94e3fd1: from storage DS-608d5eb3-8a33-4125-bd6d-2845739e66fc node DatanodeRegistration(127.0.0.1:37007, datanodeUuid=ad7930af-671f-4232-96bc-4c380afba2bb, infoPort=36867, infoSecurePort=0, ipcPort=45355, storageInfo=lv=-57;cid=testClusterID;nsid=127245085;c=1732714834229), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-27T13:40:37,815 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd76af8ca9b8693d9 with lease ID 0xa73f8761e94e3fd1: Processing first storage report for DS-dcd50d6a-300f-459d-88a0-3c6e8dd8c091 from datanode DatanodeRegistration(127.0.0.1:37007, datanodeUuid=ad7930af-671f-4232-96bc-4c380afba2bb, infoPort=36867, infoSecurePort=0, ipcPort=45355, storageInfo=lv=-57;cid=testClusterID;nsid=127245085;c=1732714834229) 2024-11-27T13:40:37,815 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd76af8ca9b8693d9 with lease ID 0xa73f8761e94e3fd1: from storage DS-dcd50d6a-300f-459d-88a0-3c6e8dd8c091 node DatanodeRegistration(127.0.0.1:37007, datanodeUuid=ad7930af-671f-4232-96bc-4c380afba2bb, infoPort=36867, infoSecurePort=0, ipcPort=45355, storageInfo=lv=-57;cid=testClusterID;nsid=127245085;c=1732714834229), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-27T13:40:37,850 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8706bd9e-726a-c9fe-5644-8f636122acec 2024-11-27T13:40:37,916 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8706bd9e-726a-c9fe-5644-8f636122acec/cluster_192d029c-90d2-1a83-1d8c-c62dd1a1bf62/zookeeper_0, clientPort=60303, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8706bd9e-726a-c9fe-5644-8f636122acec/cluster_192d029c-90d2-1a83-1d8c-c62dd1a1bf62/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8706bd9e-726a-c9fe-5644-8f636122acec/cluster_192d029c-90d2-1a83-1d8c-c62dd1a1bf62/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-27T13:40:37,928 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60303 2024-11-27T13:40:37,938 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T13:40:37,940 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T13:40:38,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37007 is added to blk_1073741825_1001 (size=7) 2024-11-27T13:40:38,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42991 is added to blk_1073741825_1001 (size=7) 2024-11-27T13:40:38,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38955 is added to blk_1073741825_1001 (size=7) 2024-11-27T13:40:38,579 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237 with version=8 2024-11-27T13:40:38,579 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/hbase-staging 2024-11-27T13:40:38,706 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-27T13:40:38,980 INFO [Time-limited test {}] client.ConnectionUtils(128): master/29b1b1ba230f:0 server-side Connection retries=45 2024-11-27T13:40:38,988 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T13:40:38,989 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-27T13:40:38,993 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-27T13:40:38,993 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T13:40:38,994 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-27T13:40:39,127 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-27T13:40:39,181 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-27T13:40:39,189 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-27T13:40:39,193 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-27T13:40:39,218 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 3161 (auto-detected) 2024-11-27T13:40:39,219 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-27T13:40:39,237 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35019 2024-11-27T13:40:39,259 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35019 connecting to ZooKeeper ensemble=127.0.0.1:60303 2024-11-27T13:40:39,384 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:350190x0, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-27T13:40:39,387 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35019-0x1017c8633370000 connected 2024-11-27T13:40:39,483 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T13:40:39,488 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T13:40:39,499 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35019-0x1017c8633370000, quorum=127.0.0.1:60303, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T13:40:39,503 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237, hbase.cluster.distributed=false 2024-11-27T13:40:39,525 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35019-0x1017c8633370000, quorum=127.0.0.1:60303, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-27T13:40:39,529 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35019 2024-11-27T13:40:39,530 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35019 2024-11-27T13:40:39,530 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35019 2024-11-27T13:40:39,531 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35019 2024-11-27T13:40:39,532 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35019 2024-11-27T13:40:39,634 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/29b1b1ba230f:0 server-side Connection retries=45 2024-11-27T13:40:39,636 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T13:40:39,636 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-27T13:40:39,636 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-27T13:40:39,636 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T13:40:39,636 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-27T13:40:39,639 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-27T13:40:39,642 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-27T13:40:39,643 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38845 2024-11-27T13:40:39,645 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38845 connecting to ZooKeeper ensemble=127.0.0.1:60303 2024-11-27T13:40:39,646 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T13:40:39,651 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T13:40:39,669 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:388450x0, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-27T13:40:39,670 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:388450x0, quorum=127.0.0.1:60303, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T13:40:39,670 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38845-0x1017c8633370001 connected 2024-11-27T13:40:39,674 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-27T13:40:39,683 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-27T13:40:39,685 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38845-0x1017c8633370001, quorum=127.0.0.1:60303, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-27T13:40:39,691 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38845-0x1017c8633370001, quorum=127.0.0.1:60303, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-27T13:40:39,692 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38845 2024-11-27T13:40:39,692 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38845 2024-11-27T13:40:39,693 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38845 2024-11-27T13:40:39,694 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38845 2024-11-27T13:40:39,694 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38845 2024-11-27T13:40:39,711 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/29b1b1ba230f:0 server-side Connection retries=45 2024-11-27T13:40:39,712 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T13:40:39,712 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-27T13:40:39,712 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-27T13:40:39,713 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T13:40:39,713 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-27T13:40:39,713 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-27T13:40:39,713 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-27T13:40:39,714 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39411 2024-11-27T13:40:39,716 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39411 connecting to ZooKeeper ensemble=127.0.0.1:60303 2024-11-27T13:40:39,717 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T13:40:39,720 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T13:40:39,732 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:394110x0, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-27T13:40:39,733 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39411-0x1017c8633370002 connected 2024-11-27T13:40:39,733 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39411-0x1017c8633370002, quorum=127.0.0.1:60303, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T13:40:39,734 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-27T13:40:39,735 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-27T13:40:39,736 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39411-0x1017c8633370002, quorum=127.0.0.1:60303, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-27T13:40:39,738 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39411-0x1017c8633370002, quorum=127.0.0.1:60303, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-27T13:40:39,741 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39411 2024-11-27T13:40:39,742 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39411 2024-11-27T13:40:39,747 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39411 2024-11-27T13:40:39,747 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39411 2024-11-27T13:40:39,748 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39411 2024-11-27T13:40:39,765 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/29b1b1ba230f:0 server-side Connection retries=45 2024-11-27T13:40:39,766 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T13:40:39,766 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-27T13:40:39,766 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-27T13:40:39,766 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T13:40:39,767 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-27T13:40:39,767 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-27T13:40:39,767 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-27T13:40:39,768 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44709 2024-11-27T13:40:39,770 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44709 connecting to ZooKeeper ensemble=127.0.0.1:60303 2024-11-27T13:40:39,771 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T13:40:39,774 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T13:40:39,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:447090x0, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-27T13:40:39,785 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:447090x0, quorum=127.0.0.1:60303, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T13:40:39,785 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44709-0x1017c8633370003 connected 2024-11-27T13:40:39,786 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-27T13:40:39,787 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-27T13:40:39,788 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44709-0x1017c8633370003, quorum=127.0.0.1:60303, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-27T13:40:39,790 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44709-0x1017c8633370003, quorum=127.0.0.1:60303, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-27T13:40:39,792 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44709 2024-11-27T13:40:39,793 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44709 2024-11-27T13:40:39,795 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44709 2024-11-27T13:40:39,796 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44709 2024-11-27T13:40:39,797 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44709 2024-11-27T13:40:39,812 DEBUG [M:0;29b1b1ba230f:35019 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;29b1b1ba230f:35019 2024-11-27T13:40:39,813 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/29b1b1ba230f,35019,1732714838827 2024-11-27T13:40:39,827 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1017c8633370001, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T13:40:39,827 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35019-0x1017c8633370000, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T13:40:39,827 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44709-0x1017c8633370003, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T13:40:39,827 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39411-0x1017c8633370002, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T13:40:39,829 DEBUG [master/29b1b1ba230f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35019-0x1017c8633370000, quorum=127.0.0.1:60303, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/29b1b1ba230f,35019,1732714838827 2024-11-27T13:40:39,858 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39411-0x1017c8633370002, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-27T13:40:39,858 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35019-0x1017c8633370000, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T13:40:39,858 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1017c8633370001, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-27T13:40:39,858 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44709-0x1017c8633370003, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-27T13:40:39,858 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39411-0x1017c8633370002, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T13:40:39,858 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1017c8633370001, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T13:40:39,858 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44709-0x1017c8633370003, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T13:40:39,859 DEBUG [master/29b1b1ba230f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35019-0x1017c8633370000, quorum=127.0.0.1:60303, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-27T13:40:39,861 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/29b1b1ba230f,35019,1732714838827 from backup master directory 2024-11-27T13:40:39,868 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44709-0x1017c8633370003, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T13:40:39,868 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39411-0x1017c8633370002, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T13:40:39,869 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1017c8633370001, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T13:40:39,869 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35019-0x1017c8633370000, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/29b1b1ba230f,35019,1732714838827 2024-11-27T13:40:39,869 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35019-0x1017c8633370000, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T13:40:39,870 WARN [master/29b1b1ba230f:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-27T13:40:39,871 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=29b1b1ba230f,35019,1732714838827 2024-11-27T13:40:39,873 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-27T13:40:39,874 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-27T13:40:39,928 DEBUG [master/29b1b1ba230f:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/hbase.id] with ID: 80a0a673-acef-4b6a-956f-a08e628106dc 2024-11-27T13:40:39,929 DEBUG [master/29b1b1ba230f:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/.tmp/hbase.id 2024-11-27T13:40:39,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38955 is added to blk_1073741826_1002 (size=42) 2024-11-27T13:40:39,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42991 is added to blk_1073741826_1002 (size=42) 2024-11-27T13:40:39,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37007 is added to blk_1073741826_1002 (size=42) 2024-11-27T13:40:39,944 DEBUG [master/29b1b1ba230f:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/.tmp/hbase.id]:[hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/hbase.id] 2024-11-27T13:40:39,990 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T13:40:39,995 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-27T13:40:40,013 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-27T13:40:40,026 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35019-0x1017c8633370000, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T13:40:40,026 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44709-0x1017c8633370003, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T13:40:40,026 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39411-0x1017c8633370002, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T13:40:40,026 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1017c8633370001, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T13:40:40,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37007 is added to blk_1073741827_1003 (size=196) 2024-11-27T13:40:40,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42991 is added to blk_1073741827_1003 (size=196) 2024-11-27T13:40:40,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38955 is added to blk_1073741827_1003 (size=196) 2024-11-27T13:40:40,064 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T13:40:40,066 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-27T13:40:40,080 DEBUG [master/29b1b1ba230f:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T13:40:40,083 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T13:40:40,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42991 is added to blk_1073741828_1004 (size=1189) 2024-11-27T13:40:40,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38955 is added to blk_1073741828_1004 (size=1189) 2024-11-27T13:40:40,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37007 is added to blk_1073741828_1004 (size=1189) 2024-11-27T13:40:40,129 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/MasterData/data/master/store 2024-11-27T13:40:40,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42991 is added to blk_1073741829_1005 (size=34) 2024-11-27T13:40:40,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38955 is added to blk_1073741829_1005 (size=34) 2024-11-27T13:40:40,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37007 is added to blk_1073741829_1005 (size=34) 2024-11-27T13:40:40,158 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-27T13:40:40,161 DEBUG [master/29b1b1ba230f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T13:40:40,163 DEBUG [master/29b1b1ba230f:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-27T13:40:40,163 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T13:40:40,163 DEBUG [master/29b1b1ba230f:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T13:40:40,165 DEBUG [master/29b1b1ba230f:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-27T13:40:40,165 DEBUG [master/29b1b1ba230f:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T13:40:40,166 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T13:40:40,167 DEBUG [master/29b1b1ba230f:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732714840163Disabling compacts and flushes for region at 1732714840163Disabling writes for close at 1732714840165 (+2 ms)Writing region close event to WAL at 1732714840165Closed at 1732714840166 (+1 ms) 2024-11-27T13:40:40,169 WARN [master/29b1b1ba230f:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/MasterData/data/master/store/.initializing 2024-11-27T13:40:40,169 DEBUG [master/29b1b1ba230f:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/MasterData/WALs/29b1b1ba230f,35019,1732714838827 2024-11-27T13:40:40,177 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T13:40:40,190 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=29b1b1ba230f%2C35019%2C1732714838827, suffix=, logDir=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/MasterData/WALs/29b1b1ba230f,35019,1732714838827, archiveDir=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/MasterData/oldWALs, maxLogs=10 2024-11-27T13:40:40,213 DEBUG [master/29b1b1ba230f:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/MasterData/WALs/29b1b1ba230f,35019,1732714838827/29b1b1ba230f%2C35019%2C1732714838827.1732714840195, exclude list is [], retry=0 2024-11-27T13:40:40,229 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37007,DS-608d5eb3-8a33-4125-bd6d-2845739e66fc,DISK] 2024-11-27T13:40:40,229 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38955,DS-fac1a99f-a8dd-4363-b07e-bd13cdba036b,DISK] 2024-11-27T13:40:40,229 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42991,DS-d60122c6-962a-4258-9fbb-6d2f24e1c6ac,DISK] 2024-11-27T13:40:40,232 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-27T13:40:40,271 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/MasterData/WALs/29b1b1ba230f,35019,1732714838827/29b1b1ba230f%2C35019%2C1732714838827.1732714840195 2024-11-27T13:40:40,271 DEBUG [master/29b1b1ba230f:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34247:34247),(127.0.0.1/127.0.0.1:36867:36867),(127.0.0.1/127.0.0.1:36243:36243)] 2024-11-27T13:40:40,272 DEBUG [master/29b1b1ba230f:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-27T13:40:40,272 DEBUG [master/29b1b1ba230f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T13:40:40,275 DEBUG [master/29b1b1ba230f:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T13:40:40,276 DEBUG [master/29b1b1ba230f:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T13:40:40,310 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T13:40:40,334 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-27T13:40:40,337 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T13:40:40,339 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T13:40:40,340 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T13:40:40,343 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-27T13:40:40,343 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T13:40:40,344 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T13:40:40,344 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T13:40:40,347 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-27T13:40:40,348 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T13:40:40,349 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T13:40:40,349 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T13:40:40,351 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-27T13:40:40,351 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T13:40:40,352 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T13:40:40,353 DEBUG [master/29b1b1ba230f:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T13:40:40,357 DEBUG [master/29b1b1ba230f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-27T13:40:40,358 DEBUG [master/29b1b1ba230f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-27T13:40:40,364 DEBUG [master/29b1b1ba230f:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T13:40:40,365 DEBUG [master/29b1b1ba230f:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T13:40:40,369 DEBUG [master/29b1b1ba230f:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-27T13:40:40,373 DEBUG [master/29b1b1ba230f:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T13:40:40,377 DEBUG [master/29b1b1ba230f:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T13:40:40,379 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67770328, jitterRate=0.009856581687927246}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-27T13:40:40,387 DEBUG [master/29b1b1ba230f:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732714840287Initializing all the Stores at 1732714840289 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732714840289Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732714840290 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732714840290Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732714840290Cleaning up temporary data from old regions at 1732714840365 (+75 ms)Region opened successfully at 1732714840387 (+22 ms) 2024-11-27T13:40:40,389 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-27T13:40:40,423 DEBUG [master/29b1b1ba230f:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4579295c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=29b1b1ba230f/172.17.0.2:0 2024-11-27T13:40:40,450 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-27T13:40:40,460 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-27T13:40:40,460 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-27T13:40:40,463 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T13:40:40,465 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-27T13:40:40,470 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-27T13:40:40,471 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-27T13:40:40,498 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-27T13:40:40,506 DEBUG [master/29b1b1ba230f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35019-0x1017c8633370000, quorum=127.0.0.1:60303, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-27T13:40:40,521 DEBUG [master/29b1b1ba230f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-27T13:40:40,523 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-27T13:40:40,525 DEBUG [master/29b1b1ba230f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35019-0x1017c8633370000, quorum=127.0.0.1:60303, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-27T13:40:40,538 DEBUG [master/29b1b1ba230f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-27T13:40:40,541 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-27T13:40:40,545 DEBUG [master/29b1b1ba230f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35019-0x1017c8633370000, quorum=127.0.0.1:60303, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-27T13:40:40,552 DEBUG [master/29b1b1ba230f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-27T13:40:40,554 DEBUG [master/29b1b1ba230f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35019-0x1017c8633370000, quorum=127.0.0.1:60303, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-27T13:40:40,563 DEBUG [master/29b1b1ba230f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-27T13:40:40,584 DEBUG [master/29b1b1ba230f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35019-0x1017c8633370000, quorum=127.0.0.1:60303, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-27T13:40:40,594 DEBUG [master/29b1b1ba230f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-27T13:40:40,605 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35019-0x1017c8633370000, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-27T13:40:40,605 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39411-0x1017c8633370002, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-27T13:40:40,605 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44709-0x1017c8633370003, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-27T13:40:40,605 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39411-0x1017c8633370002, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T13:40:40,605 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44709-0x1017c8633370003, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T13:40:40,606 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1017c8633370001, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-27T13:40:40,606 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1017c8633370001, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T13:40:40,606 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35019-0x1017c8633370000, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T13:40:40,610 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=29b1b1ba230f,35019,1732714838827, sessionid=0x1017c8633370000, setting cluster-up flag (Was=false) 2024-11-27T13:40:40,644 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39411-0x1017c8633370002, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T13:40:40,644 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35019-0x1017c8633370000, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T13:40:40,644 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44709-0x1017c8633370003, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T13:40:40,644 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1017c8633370001, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T13:40:40,679 DEBUG [master/29b1b1ba230f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-27T13:40:40,683 DEBUG [master/29b1b1ba230f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=29b1b1ba230f,35019,1732714838827 2024-11-27T13:40:40,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35019-0x1017c8633370000, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T13:40:40,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1017c8633370001, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T13:40:40,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44709-0x1017c8633370003, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T13:40:40,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39411-0x1017c8633370002, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T13:40:40,742 DEBUG [master/29b1b1ba230f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-27T13:40:40,745 DEBUG [master/29b1b1ba230f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=29b1b1ba230f,35019,1732714838827 2024-11-27T13:40:40,755 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-27T13:40:40,800 INFO [RS:0;29b1b1ba230f:38845 {}] regionserver.HRegionServer(746): ClusterId : 80a0a673-acef-4b6a-956f-a08e628106dc 2024-11-27T13:40:40,800 INFO [RS:1;29b1b1ba230f:39411 {}] regionserver.HRegionServer(746): ClusterId : 80a0a673-acef-4b6a-956f-a08e628106dc 2024-11-27T13:40:40,800 INFO [RS:2;29b1b1ba230f:44709 {}] regionserver.HRegionServer(746): ClusterId : 80a0a673-acef-4b6a-956f-a08e628106dc 2024-11-27T13:40:40,803 DEBUG [RS:2;29b1b1ba230f:44709 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-27T13:40:40,803 DEBUG [RS:0;29b1b1ba230f:38845 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-27T13:40:40,803 DEBUG [RS:1;29b1b1ba230f:39411 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-27T13:40:40,818 DEBUG [RS:1;29b1b1ba230f:39411 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-27T13:40:40,818 DEBUG [RS:0;29b1b1ba230f:38845 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-27T13:40:40,818 DEBUG [RS:0;29b1b1ba230f:38845 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-27T13:40:40,818 DEBUG [RS:1;29b1b1ba230f:39411 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-27T13:40:40,820 DEBUG [RS:2;29b1b1ba230f:44709 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-27T13:40:40,820 DEBUG [RS:2;29b1b1ba230f:44709 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-27T13:40:40,821 DEBUG [master/29b1b1ba230f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T13:40:40,830 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-27T13:40:40,837 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-27T13:40:40,838 DEBUG [RS:1;29b1b1ba230f:39411 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-27T13:40:40,838 DEBUG [RS:0;29b1b1ba230f:38845 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-27T13:40:40,838 DEBUG [RS:2;29b1b1ba230f:44709 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-27T13:40:40,838 DEBUG [RS:0;29b1b1ba230f:38845 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c412c99, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=29b1b1ba230f/172.17.0.2:0 2024-11-27T13:40:40,838 DEBUG [RS:2;29b1b1ba230f:44709 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bed3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=29b1b1ba230f/172.17.0.2:0 2024-11-27T13:40:40,838 DEBUG [RS:1;29b1b1ba230f:39411 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49ea214c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=29b1b1ba230f/172.17.0.2:0 2024-11-27T13:40:40,843 DEBUG [master/29b1b1ba230f:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 29b1b1ba230f,35019,1732714838827 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-27T13:40:40,851 DEBUG [master/29b1b1ba230f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/29b1b1ba230f:0, corePoolSize=5, maxPoolSize=5 2024-11-27T13:40:40,851 DEBUG [master/29b1b1ba230f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/29b1b1ba230f:0, corePoolSize=5, maxPoolSize=5 2024-11-27T13:40:40,851 DEBUG [master/29b1b1ba230f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/29b1b1ba230f:0, corePoolSize=5, maxPoolSize=5 2024-11-27T13:40:40,851 DEBUG [master/29b1b1ba230f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/29b1b1ba230f:0, corePoolSize=5, maxPoolSize=5 2024-11-27T13:40:40,851 DEBUG [master/29b1b1ba230f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/29b1b1ba230f:0, corePoolSize=10, maxPoolSize=10 2024-11-27T13:40:40,851 DEBUG [master/29b1b1ba230f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/29b1b1ba230f:0, corePoolSize=1, maxPoolSize=1 2024-11-27T13:40:40,852 DEBUG [master/29b1b1ba230f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/29b1b1ba230f:0, corePoolSize=2, maxPoolSize=2 2024-11-27T13:40:40,852 DEBUG [master/29b1b1ba230f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/29b1b1ba230f:0, corePoolSize=1, maxPoolSize=1 2024-11-27T13:40:40,858 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732714870858 2024-11-27T13:40:40,859 DEBUG [RS:1;29b1b1ba230f:39411 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;29b1b1ba230f:39411 2024-11-27T13:40:40,859 DEBUG [RS:2;29b1b1ba230f:44709 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;29b1b1ba230f:44709 2024-11-27T13:40:40,859 DEBUG [RS:0;29b1b1ba230f:38845 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;29b1b1ba230f:38845 2024-11-27T13:40:40,860 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-27T13:40:40,861 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-27T13:40:40,863 INFO [RS:1;29b1b1ba230f:39411 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-27T13:40:40,863 INFO [RS:0;29b1b1ba230f:38845 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-27T13:40:40,863 INFO [RS:2;29b1b1ba230f:44709 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-27T13:40:40,863 INFO [RS:2;29b1b1ba230f:44709 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-27T13:40:40,863 INFO [RS:0;29b1b1ba230f:38845 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-27T13:40:40,863 INFO [RS:1;29b1b1ba230f:39411 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-27T13:40:40,863 DEBUG [RS:0;29b1b1ba230f:38845 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-27T13:40:40,863 DEBUG [RS:2;29b1b1ba230f:44709 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-27T13:40:40,863 DEBUG [RS:1;29b1b1ba230f:39411 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-27T13:40:40,863 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-27T13:40:40,864 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-27T13:40:40,866 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-27T13:40:40,866 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-27T13:40:40,866 INFO [RS:2;29b1b1ba230f:44709 {}] regionserver.HRegionServer(2659): reportForDuty to master=29b1b1ba230f,35019,1732714838827 with port=44709, startcode=1732714839765 2024-11-27T13:40:40,866 INFO [RS:0;29b1b1ba230f:38845 {}] regionserver.HRegionServer(2659): reportForDuty to master=29b1b1ba230f,35019,1732714838827 with port=38845, startcode=1732714839599 2024-11-27T13:40:40,866 INFO [RS:1;29b1b1ba230f:39411 {}] regionserver.HRegionServer(2659): reportForDuty to master=29b1b1ba230f,35019,1732714838827 with port=39411, startcode=1732714839711 2024-11-27T13:40:40,866 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-27T13:40:40,867 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-27T13:40:40,867 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-27T13:40:40,871 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T13:40:40,872 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-27T13:40:40,872 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-27T13:40:40,873 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-27T13:40:40,874 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-27T13:40:40,876 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-27T13:40:40,876 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-27T13:40:40,878 DEBUG [master/29b1b1ba230f:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/29b1b1ba230f:0:becomeActiveMaster-HFileCleaner.large.0-1732714840878,5,FailOnTimeoutGroup] 2024-11-27T13:40:40,881 DEBUG [RS:0;29b1b1ba230f:38845 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-27T13:40:40,881 DEBUG [RS:2;29b1b1ba230f:44709 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-27T13:40:40,881 DEBUG [RS:1;29b1b1ba230f:39411 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-27T13:40:40,884 DEBUG [master/29b1b1ba230f:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/29b1b1ba230f:0:becomeActiveMaster-HFileCleaner.small.0-1732714840879,5,FailOnTimeoutGroup] 2024-11-27T13:40:40,884 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-27T13:40:40,885 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-27T13:40:40,886 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-27T13:40:40,887 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-27T13:40:40,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37007 is added to blk_1073741831_1007 (size=1321) 2024-11-27T13:40:40,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42991 is added to blk_1073741831_1007 (size=1321) 2024-11-27T13:40:40,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38955 is added to blk_1073741831_1007 (size=1321) 2024-11-27T13:40:40,896 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-27T13:40:40,897 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237 2024-11-27T13:40:40,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38955 is added to blk_1073741832_1008 (size=32) 2024-11-27T13:40:40,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37007 is added to blk_1073741832_1008 (size=32) 2024-11-27T13:40:40,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42991 is added to blk_1073741832_1008 (size=32) 2024-11-27T13:40:40,944 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56581, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-27T13:40:40,944 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59605, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-27T13:40:40,944 INFO [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46241, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-27T13:40:40,950 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35019 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:40,953 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35019 {}] master.ServerManager(517): Registering regionserver=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:40,963 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35019 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 29b1b1ba230f,39411,1732714839711 2024-11-27T13:40:40,964 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35019 {}] master.ServerManager(517): Registering regionserver=29b1b1ba230f,39411,1732714839711 2024-11-27T13:40:40,968 DEBUG [RS:2;29b1b1ba230f:44709 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237 2024-11-27T13:40:40,968 DEBUG [RS:2;29b1b1ba230f:44709 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43491 2024-11-27T13:40:40,968 DEBUG [RS:2;29b1b1ba230f:44709 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-27T13:40:40,969 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35019 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:40,969 DEBUG [RS:1;29b1b1ba230f:39411 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237 2024-11-27T13:40:40,969 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35019 {}] master.ServerManager(517): Registering regionserver=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:40,969 DEBUG [RS:1;29b1b1ba230f:39411 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43491 2024-11-27T13:40:40,970 DEBUG [RS:1;29b1b1ba230f:39411 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-27T13:40:40,973 DEBUG [RS:0;29b1b1ba230f:38845 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237 2024-11-27T13:40:40,973 DEBUG [RS:0;29b1b1ba230f:38845 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43491 2024-11-27T13:40:40,974 DEBUG [RS:0;29b1b1ba230f:38845 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-27T13:40:40,980 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35019-0x1017c8633370000, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-27T13:40:41,044 DEBUG [RS:1;29b1b1ba230f:39411 {}] zookeeper.ZKUtil(111): regionserver:39411-0x1017c8633370002, quorum=127.0.0.1:60303, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/29b1b1ba230f,39411,1732714839711 2024-11-27T13:40:41,044 WARN [RS:1;29b1b1ba230f:39411 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-27T13:40:41,044 DEBUG [RS:0;29b1b1ba230f:38845 {}] zookeeper.ZKUtil(111): regionserver:38845-0x1017c8633370001, quorum=127.0.0.1:60303, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:41,044 DEBUG [RS:2;29b1b1ba230f:44709 {}] zookeeper.ZKUtil(111): regionserver:44709-0x1017c8633370003, quorum=127.0.0.1:60303, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:41,044 WARN [RS:0;29b1b1ba230f:38845 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-27T13:40:41,044 WARN [RS:2;29b1b1ba230f:44709 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-27T13:40:41,045 INFO [RS:1;29b1b1ba230f:39411 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T13:40:41,045 INFO [RS:0;29b1b1ba230f:38845 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T13:40:41,045 INFO [RS:2;29b1b1ba230f:44709 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T13:40:41,045 DEBUG [RS:0;29b1b1ba230f:38845 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/WALs/29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:41,045 DEBUG [RS:1;29b1b1ba230f:39411 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/WALs/29b1b1ba230f,39411,1732714839711 2024-11-27T13:40:41,045 DEBUG [RS:2;29b1b1ba230f:44709 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/WALs/29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:41,046 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [29b1b1ba230f,38845,1732714839599] 2024-11-27T13:40:41,046 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [29b1b1ba230f,44709,1732714839765] 2024-11-27T13:40:41,047 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [29b1b1ba230f,39411,1732714839711] 2024-11-27T13:40:41,071 INFO [RS:2;29b1b1ba230f:44709 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-27T13:40:41,071 INFO [RS:0;29b1b1ba230f:38845 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-27T13:40:41,071 INFO [RS:1;29b1b1ba230f:39411 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-27T13:40:41,088 INFO [RS:2;29b1b1ba230f:44709 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-27T13:40:41,088 INFO [RS:0;29b1b1ba230f:38845 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-27T13:40:41,088 INFO [RS:1;29b1b1ba230f:39411 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-27T13:40:41,093 INFO [RS:1;29b1b1ba230f:39411 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-27T13:40:41,094 INFO [RS:2;29b1b1ba230f:44709 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-27T13:40:41,094 INFO [RS:0;29b1b1ba230f:38845 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-27T13:40:41,094 INFO [RS:1;29b1b1ba230f:39411 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T13:40:41,094 INFO [RS:2;29b1b1ba230f:44709 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T13:40:41,094 INFO [RS:0;29b1b1ba230f:38845 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T13:40:41,095 INFO [RS:1;29b1b1ba230f:39411 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-27T13:40:41,099 INFO [RS:2;29b1b1ba230f:44709 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-27T13:40:41,099 INFO [RS:0;29b1b1ba230f:38845 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-27T13:40:41,101 INFO [RS:1;29b1b1ba230f:39411 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-27T13:40:41,101 INFO [RS:0;29b1b1ba230f:38845 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-27T13:40:41,102 INFO [RS:2;29b1b1ba230f:44709 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-27T13:40:41,103 INFO [RS:1;29b1b1ba230f:39411 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-27T13:40:41,103 INFO [RS:2;29b1b1ba230f:44709 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-27T13:40:41,103 INFO [RS:0;29b1b1ba230f:38845 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-27T13:40:41,103 DEBUG [RS:2;29b1b1ba230f:44709 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/29b1b1ba230f:0, corePoolSize=1, maxPoolSize=1 2024-11-27T13:40:41,103 DEBUG [RS:0;29b1b1ba230f:38845 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/29b1b1ba230f:0, corePoolSize=1, maxPoolSize=1 2024-11-27T13:40:41,103 DEBUG [RS:1;29b1b1ba230f:39411 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/29b1b1ba230f:0, corePoolSize=1, maxPoolSize=1 2024-11-27T13:40:41,104 DEBUG [RS:2;29b1b1ba230f:44709 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/29b1b1ba230f:0, corePoolSize=1, maxPoolSize=1 2024-11-27T13:40:41,104 DEBUG [RS:0;29b1b1ba230f:38845 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/29b1b1ba230f:0, corePoolSize=1, maxPoolSize=1 2024-11-27T13:40:41,104 DEBUG [RS:1;29b1b1ba230f:39411 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/29b1b1ba230f:0, corePoolSize=1, maxPoolSize=1 2024-11-27T13:40:41,104 DEBUG [RS:2;29b1b1ba230f:44709 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/29b1b1ba230f:0, corePoolSize=1, maxPoolSize=1 2024-11-27T13:40:41,104 DEBUG [RS:0;29b1b1ba230f:38845 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/29b1b1ba230f:0, corePoolSize=1, maxPoolSize=1 2024-11-27T13:40:41,104 DEBUG [RS:1;29b1b1ba230f:39411 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/29b1b1ba230f:0, corePoolSize=1, maxPoolSize=1 2024-11-27T13:40:41,104 DEBUG [RS:2;29b1b1ba230f:44709 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/29b1b1ba230f:0, corePoolSize=1, maxPoolSize=1 2024-11-27T13:40:41,104 DEBUG [RS:0;29b1b1ba230f:38845 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/29b1b1ba230f:0, corePoolSize=1, maxPoolSize=1 2024-11-27T13:40:41,104 DEBUG [RS:1;29b1b1ba230f:39411 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/29b1b1ba230f:0, corePoolSize=1, maxPoolSize=1 2024-11-27T13:40:41,104 DEBUG [RS:2;29b1b1ba230f:44709 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/29b1b1ba230f:0, corePoolSize=1, maxPoolSize=1 2024-11-27T13:40:41,104 DEBUG [RS:0;29b1b1ba230f:38845 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/29b1b1ba230f:0, corePoolSize=1, maxPoolSize=1 2024-11-27T13:40:41,104 DEBUG [RS:1;29b1b1ba230f:39411 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/29b1b1ba230f:0, corePoolSize=1, maxPoolSize=1 2024-11-27T13:40:41,104 DEBUG [RS:2;29b1b1ba230f:44709 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/29b1b1ba230f:0, corePoolSize=2, maxPoolSize=2 2024-11-27T13:40:41,104 DEBUG [RS:1;29b1b1ba230f:39411 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/29b1b1ba230f:0, corePoolSize=2, maxPoolSize=2 2024-11-27T13:40:41,104 DEBUG [RS:0;29b1b1ba230f:38845 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/29b1b1ba230f:0, corePoolSize=2, maxPoolSize=2 2024-11-27T13:40:41,104 DEBUG [RS:2;29b1b1ba230f:44709 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/29b1b1ba230f:0, corePoolSize=1, maxPoolSize=1 2024-11-27T13:40:41,104 DEBUG [RS:0;29b1b1ba230f:38845 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/29b1b1ba230f:0, corePoolSize=1, maxPoolSize=1 2024-11-27T13:40:41,104 DEBUG [RS:1;29b1b1ba230f:39411 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/29b1b1ba230f:0, corePoolSize=1, maxPoolSize=1 2024-11-27T13:40:41,105 DEBUG [RS:2;29b1b1ba230f:44709 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/29b1b1ba230f:0, corePoolSize=1, maxPoolSize=1 2024-11-27T13:40:41,105 DEBUG [RS:0;29b1b1ba230f:38845 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/29b1b1ba230f:0, corePoolSize=1, maxPoolSize=1 2024-11-27T13:40:41,105 DEBUG [RS:1;29b1b1ba230f:39411 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/29b1b1ba230f:0, corePoolSize=1, maxPoolSize=1 2024-11-27T13:40:41,105 DEBUG [RS:2;29b1b1ba230f:44709 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/29b1b1ba230f:0, corePoolSize=1, maxPoolSize=1 2024-11-27T13:40:41,105 DEBUG [RS:0;29b1b1ba230f:38845 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/29b1b1ba230f:0, corePoolSize=1, maxPoolSize=1 2024-11-27T13:40:41,105 DEBUG [RS:1;29b1b1ba230f:39411 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/29b1b1ba230f:0, corePoolSize=1, maxPoolSize=1 2024-11-27T13:40:41,105 DEBUG [RS:2;29b1b1ba230f:44709 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/29b1b1ba230f:0, corePoolSize=1, maxPoolSize=1 2024-11-27T13:40:41,105 DEBUG [RS:0;29b1b1ba230f:38845 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/29b1b1ba230f:0, corePoolSize=1, maxPoolSize=1 2024-11-27T13:40:41,105 DEBUG [RS:1;29b1b1ba230f:39411 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/29b1b1ba230f:0, corePoolSize=1, maxPoolSize=1 2024-11-27T13:40:41,105 DEBUG [RS:0;29b1b1ba230f:38845 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/29b1b1ba230f:0, corePoolSize=1, maxPoolSize=1 2024-11-27T13:40:41,105 DEBUG [RS:1;29b1b1ba230f:39411 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/29b1b1ba230f:0, corePoolSize=1, maxPoolSize=1 2024-11-27T13:40:41,105 DEBUG [RS:2;29b1b1ba230f:44709 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/29b1b1ba230f:0, corePoolSize=1, maxPoolSize=1 2024-11-27T13:40:41,105 DEBUG [RS:0;29b1b1ba230f:38845 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/29b1b1ba230f:0, corePoolSize=1, maxPoolSize=1 2024-11-27T13:40:41,105 DEBUG [RS:1;29b1b1ba230f:39411 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/29b1b1ba230f:0, corePoolSize=1, maxPoolSize=1 2024-11-27T13:40:41,105 DEBUG [RS:2;29b1b1ba230f:44709 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/29b1b1ba230f:0, corePoolSize=1, maxPoolSize=1 2024-11-27T13:40:41,105 DEBUG [RS:0;29b1b1ba230f:38845 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/29b1b1ba230f:0, corePoolSize=3, maxPoolSize=3 2024-11-27T13:40:41,105 DEBUG [RS:1;29b1b1ba230f:39411 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/29b1b1ba230f:0, corePoolSize=3, maxPoolSize=3 2024-11-27T13:40:41,105 DEBUG [RS:2;29b1b1ba230f:44709 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/29b1b1ba230f:0, corePoolSize=3, maxPoolSize=3 2024-11-27T13:40:41,106 DEBUG [RS:0;29b1b1ba230f:38845 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/29b1b1ba230f:0, corePoolSize=3, maxPoolSize=3 2024-11-27T13:40:41,106 DEBUG [RS:2;29b1b1ba230f:44709 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/29b1b1ba230f:0, corePoolSize=3, maxPoolSize=3 2024-11-27T13:40:41,106 DEBUG [RS:1;29b1b1ba230f:39411 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/29b1b1ba230f:0, corePoolSize=3, maxPoolSize=3 2024-11-27T13:40:41,108 INFO [RS:0;29b1b1ba230f:38845 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T13:40:41,108 INFO [RS:0;29b1b1ba230f:38845 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T13:40:41,108 INFO [RS:0;29b1b1ba230f:38845 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T13:40:41,108 INFO [RS:0;29b1b1ba230f:38845 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-27T13:40:41,109 INFO [RS:0;29b1b1ba230f:38845 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-27T13:40:41,109 INFO [RS:0;29b1b1ba230f:38845 {}] hbase.ChoreService(168): Chore ScheduledChore name=29b1b1ba230f,38845,1732714839599-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-27T13:40:41,109 INFO [RS:2;29b1b1ba230f:44709 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T13:40:41,110 INFO [RS:2;29b1b1ba230f:44709 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T13:40:41,110 INFO [RS:2;29b1b1ba230f:44709 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T13:40:41,110 INFO [RS:2;29b1b1ba230f:44709 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-27T13:40:41,110 INFO [RS:2;29b1b1ba230f:44709 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-27T13:40:41,110 INFO [RS:2;29b1b1ba230f:44709 {}] hbase.ChoreService(168): Chore ScheduledChore name=29b1b1ba230f,44709,1732714839765-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-27T13:40:41,111 INFO [RS:1;29b1b1ba230f:39411 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T13:40:41,112 INFO [RS:1;29b1b1ba230f:39411 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T13:40:41,112 INFO [RS:1;29b1b1ba230f:39411 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T13:40:41,112 INFO [RS:1;29b1b1ba230f:39411 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-27T13:40:41,112 INFO [RS:1;29b1b1ba230f:39411 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-27T13:40:41,112 INFO [RS:1;29b1b1ba230f:39411 {}] hbase.ChoreService(168): Chore ScheduledChore name=29b1b1ba230f,39411,1732714839711-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-27T13:40:41,131 INFO [RS:2;29b1b1ba230f:44709 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-27T13:40:41,133 INFO [RS:2;29b1b1ba230f:44709 {}] hbase.ChoreService(168): Chore ScheduledChore name=29b1b1ba230f,44709,1732714839765-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T13:40:41,133 INFO [RS:2;29b1b1ba230f:44709 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T13:40:41,133 INFO [RS:2;29b1b1ba230f:44709 {}] regionserver.Replication(171): 29b1b1ba230f,44709,1732714839765 started 2024-11-27T13:40:41,134 INFO [RS:0;29b1b1ba230f:38845 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-27T13:40:41,134 INFO [RS:0;29b1b1ba230f:38845 {}] hbase.ChoreService(168): Chore ScheduledChore name=29b1b1ba230f,38845,1732714839599-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T13:40:41,134 INFO [RS:0;29b1b1ba230f:38845 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T13:40:41,134 INFO [RS:0;29b1b1ba230f:38845 {}] regionserver.Replication(171): 29b1b1ba230f,38845,1732714839599 started 2024-11-27T13:40:41,135 INFO [RS:1;29b1b1ba230f:39411 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-27T13:40:41,135 INFO [RS:1;29b1b1ba230f:39411 {}] hbase.ChoreService(168): Chore ScheduledChore name=29b1b1ba230f,39411,1732714839711-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T13:40:41,135 INFO [RS:1;29b1b1ba230f:39411 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T13:40:41,136 INFO [RS:1;29b1b1ba230f:39411 {}] regionserver.Replication(171): 29b1b1ba230f,39411,1732714839711 started 2024-11-27T13:40:41,153 INFO [RS:2;29b1b1ba230f:44709 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T13:40:41,153 INFO [RS:2;29b1b1ba230f:44709 {}] regionserver.HRegionServer(1482): Serving as 29b1b1ba230f,44709,1732714839765, RpcServer on 29b1b1ba230f/172.17.0.2:44709, sessionid=0x1017c8633370003 2024-11-27T13:40:41,155 DEBUG [RS:2;29b1b1ba230f:44709 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-27T13:40:41,155 DEBUG [RS:2;29b1b1ba230f:44709 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:41,155 DEBUG [RS:2;29b1b1ba230f:44709 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '29b1b1ba230f,44709,1732714839765' 2024-11-27T13:40:41,155 DEBUG [RS:2;29b1b1ba230f:44709 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-27T13:40:41,156 DEBUG [RS:2;29b1b1ba230f:44709 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-27T13:40:41,157 DEBUG [RS:2;29b1b1ba230f:44709 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-27T13:40:41,157 DEBUG [RS:2;29b1b1ba230f:44709 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-27T13:40:41,157 DEBUG [RS:2;29b1b1ba230f:44709 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:41,158 DEBUG [RS:2;29b1b1ba230f:44709 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '29b1b1ba230f,44709,1732714839765' 2024-11-27T13:40:41,158 DEBUG [RS:2;29b1b1ba230f:44709 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-27T13:40:41,158 INFO [RS:0;29b1b1ba230f:38845 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T13:40:41,158 INFO [RS:1;29b1b1ba230f:39411 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T13:40:41,159 INFO [RS:0;29b1b1ba230f:38845 {}] regionserver.HRegionServer(1482): Serving as 29b1b1ba230f,38845,1732714839599, RpcServer on 29b1b1ba230f/172.17.0.2:38845, sessionid=0x1017c8633370001 2024-11-27T13:40:41,159 INFO [RS:1;29b1b1ba230f:39411 {}] regionserver.HRegionServer(1482): Serving as 29b1b1ba230f,39411,1732714839711, RpcServer on 29b1b1ba230f/172.17.0.2:39411, sessionid=0x1017c8633370002 2024-11-27T13:40:41,159 DEBUG [RS:0;29b1b1ba230f:38845 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-27T13:40:41,159 DEBUG [RS:1;29b1b1ba230f:39411 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-27T13:40:41,159 DEBUG [RS:0;29b1b1ba230f:38845 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:41,159 DEBUG [RS:1;29b1b1ba230f:39411 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 29b1b1ba230f,39411,1732714839711 2024-11-27T13:40:41,159 DEBUG [RS:1;29b1b1ba230f:39411 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '29b1b1ba230f,39411,1732714839711' 2024-11-27T13:40:41,159 DEBUG [RS:0;29b1b1ba230f:38845 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '29b1b1ba230f,38845,1732714839599' 2024-11-27T13:40:41,159 DEBUG [RS:2;29b1b1ba230f:44709 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-27T13:40:41,159 DEBUG [RS:1;29b1b1ba230f:39411 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-27T13:40:41,159 DEBUG [RS:0;29b1b1ba230f:38845 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-27T13:40:41,160 DEBUG [RS:2;29b1b1ba230f:44709 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-27T13:40:41,160 INFO [RS:2;29b1b1ba230f:44709 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-27T13:40:41,160 DEBUG [RS:0;29b1b1ba230f:38845 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-27T13:40:41,160 INFO [RS:2;29b1b1ba230f:44709 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-27T13:40:41,160 DEBUG [RS:1;29b1b1ba230f:39411 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-27T13:40:41,161 DEBUG [RS:0;29b1b1ba230f:38845 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-27T13:40:41,161 DEBUG [RS:0;29b1b1ba230f:38845 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-27T13:40:41,161 DEBUG [RS:1;29b1b1ba230f:39411 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-27T13:40:41,161 DEBUG [RS:0;29b1b1ba230f:38845 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:41,161 DEBUG [RS:1;29b1b1ba230f:39411 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-27T13:40:41,161 DEBUG [RS:0;29b1b1ba230f:38845 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '29b1b1ba230f,38845,1732714839599' 2024-11-27T13:40:41,161 DEBUG [RS:1;29b1b1ba230f:39411 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 29b1b1ba230f,39411,1732714839711 2024-11-27T13:40:41,161 DEBUG [RS:0;29b1b1ba230f:38845 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-27T13:40:41,161 DEBUG [RS:1;29b1b1ba230f:39411 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '29b1b1ba230f,39411,1732714839711' 2024-11-27T13:40:41,161 DEBUG [RS:1;29b1b1ba230f:39411 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-27T13:40:41,162 DEBUG [RS:0;29b1b1ba230f:38845 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-27T13:40:41,162 DEBUG [RS:1;29b1b1ba230f:39411 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-27T13:40:41,162 DEBUG [RS:0;29b1b1ba230f:38845 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-27T13:40:41,162 INFO [RS:0;29b1b1ba230f:38845 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-27T13:40:41,162 DEBUG [RS:1;29b1b1ba230f:39411 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-27T13:40:41,162 INFO [RS:0;29b1b1ba230f:38845 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-27T13:40:41,162 INFO [RS:1;29b1b1ba230f:39411 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-27T13:40:41,163 INFO [RS:1;29b1b1ba230f:39411 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-27T13:40:41,265 INFO [RS:2;29b1b1ba230f:44709 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T13:40:41,265 INFO [RS:1;29b1b1ba230f:39411 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T13:40:41,266 INFO [RS:0;29b1b1ba230f:38845 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T13:40:41,269 INFO [RS:2;29b1b1ba230f:44709 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=29b1b1ba230f%2C44709%2C1732714839765, suffix=, logDir=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/WALs/29b1b1ba230f,44709,1732714839765, archiveDir=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/oldWALs, maxLogs=32 2024-11-27T13:40:41,269 INFO [RS:0;29b1b1ba230f:38845 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=29b1b1ba230f%2C38845%2C1732714839599, suffix=, logDir=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/WALs/29b1b1ba230f,38845,1732714839599, archiveDir=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/oldWALs, maxLogs=32 2024-11-27T13:40:41,269 INFO [RS:1;29b1b1ba230f:39411 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=29b1b1ba230f%2C39411%2C1732714839711, suffix=, logDir=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/WALs/29b1b1ba230f,39411,1732714839711, archiveDir=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/oldWALs, maxLogs=32 2024-11-27T13:40:41,286 DEBUG [RS:1;29b1b1ba230f:39411 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/WALs/29b1b1ba230f,39411,1732714839711/29b1b1ba230f%2C39411%2C1732714839711.1732714841272, exclude list is [], retry=0 2024-11-27T13:40:41,291 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37007,DS-608d5eb3-8a33-4125-bd6d-2845739e66fc,DISK] 2024-11-27T13:40:41,291 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38955,DS-fac1a99f-a8dd-4363-b07e-bd13cdba036b,DISK] 2024-11-27T13:40:41,291 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42991,DS-d60122c6-962a-4258-9fbb-6d2f24e1c6ac,DISK] 2024-11-27T13:40:41,292 DEBUG [RS:0;29b1b1ba230f:38845 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/WALs/29b1b1ba230f,38845,1732714839599/29b1b1ba230f%2C38845%2C1732714839599.1732714841273, exclude list is [], retry=0 2024-11-27T13:40:41,292 DEBUG [RS:2;29b1b1ba230f:44709 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/WALs/29b1b1ba230f,44709,1732714839765/29b1b1ba230f%2C44709%2C1732714839765.1732714841273, exclude list is [], retry=0 2024-11-27T13:40:41,297 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38955,DS-fac1a99f-a8dd-4363-b07e-bd13cdba036b,DISK] 2024-11-27T13:40:41,297 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38955,DS-fac1a99f-a8dd-4363-b07e-bd13cdba036b,DISK] 2024-11-27T13:40:41,297 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37007,DS-608d5eb3-8a33-4125-bd6d-2845739e66fc,DISK] 2024-11-27T13:40:41,298 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42991,DS-d60122c6-962a-4258-9fbb-6d2f24e1c6ac,DISK] 2024-11-27T13:40:41,298 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42991,DS-d60122c6-962a-4258-9fbb-6d2f24e1c6ac,DISK] 2024-11-27T13:40:41,298 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37007,DS-608d5eb3-8a33-4125-bd6d-2845739e66fc,DISK] 2024-11-27T13:40:41,305 INFO [RS:1;29b1b1ba230f:39411 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/WALs/29b1b1ba230f,39411,1732714839711/29b1b1ba230f%2C39411%2C1732714839711.1732714841272 2024-11-27T13:40:41,306 DEBUG [RS:1;29b1b1ba230f:39411 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36867:36867),(127.0.0.1/127.0.0.1:36243:36243),(127.0.0.1/127.0.0.1:34247:34247)] 2024-11-27T13:40:41,313 INFO [RS:0;29b1b1ba230f:38845 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/WALs/29b1b1ba230f,38845,1732714839599/29b1b1ba230f%2C38845%2C1732714839599.1732714841273 2024-11-27T13:40:41,313 INFO [RS:2;29b1b1ba230f:44709 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/WALs/29b1b1ba230f,44709,1732714839765/29b1b1ba230f%2C44709%2C1732714839765.1732714841273 2024-11-27T13:40:41,313 DEBUG [RS:2;29b1b1ba230f:44709 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36243:36243),(127.0.0.1/127.0.0.1:34247:34247),(127.0.0.1/127.0.0.1:36867:36867)] 2024-11-27T13:40:41,313 DEBUG [RS:0;29b1b1ba230f:38845 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36867:36867),(127.0.0.1/127.0.0.1:34247:34247),(127.0.0.1/127.0.0.1:36243:36243)] 2024-11-27T13:40:41,345 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T13:40:41,348 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-27T13:40:41,350 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-27T13:40:41,350 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T13:40:41,351 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T13:40:41,351 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-27T13:40:41,354 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-27T13:40:41,354 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T13:40:41,355 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T13:40:41,355 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-27T13:40:41,358 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-27T13:40:41,358 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T13:40:41,359 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T13:40:41,359 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-27T13:40:41,362 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-27T13:40:41,362 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T13:40:41,363 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T13:40:41,363 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-27T13:40:41,365 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/hbase/meta/1588230740 2024-11-27T13:40:41,365 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/hbase/meta/1588230740 2024-11-27T13:40:41,368 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-27T13:40:41,368 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-27T13:40:41,369 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-27T13:40:41,372 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-27T13:40:41,375 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T13:40:41,376 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60459472, jitterRate=-0.09908366203308105}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-27T13:40:41,377 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732714841346Initializing all the Stores at 1732714841347 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732714841347Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732714841348 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732714841348Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732714841348Cleaning up temporary data from old regions at 1732714841368 (+20 ms)Region opened successfully at 1732714841377 (+9 ms) 2024-11-27T13:40:41,378 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-27T13:40:41,378 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-27T13:40:41,378 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-27T13:40:41,378 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-27T13:40:41,378 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-27T13:40:41,379 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-27T13:40:41,379 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732714841378Disabling compacts and flushes for region at 1732714841378Disabling writes for close at 1732714841378Writing region close event to WAL at 1732714841379 (+1 ms)Closed at 1732714841379 2024-11-27T13:40:41,382 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-27T13:40:41,382 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-27T13:40:41,389 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-27T13:40:41,399 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T13:40:41,402 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-27T13:40:41,555 DEBUG [29b1b1ba230f:35019 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-27T13:40:41,564 DEBUG [29b1b1ba230f:35019 {}] balancer.BalancerClusterState(204): Hosts are {29b1b1ba230f=0} racks are {/default-rack=0} 2024-11-27T13:40:41,570 DEBUG [29b1b1ba230f:35019 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-27T13:40:41,570 DEBUG [29b1b1ba230f:35019 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-27T13:40:41,570 DEBUG [29b1b1ba230f:35019 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-27T13:40:41,570 DEBUG [29b1b1ba230f:35019 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-27T13:40:41,570 DEBUG [29b1b1ba230f:35019 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-27T13:40:41,570 DEBUG [29b1b1ba230f:35019 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-27T13:40:41,570 INFO [29b1b1ba230f:35019 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-27T13:40:41,570 INFO [29b1b1ba230f:35019 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-27T13:40:41,570 INFO [29b1b1ba230f:35019 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-27T13:40:41,571 DEBUG [29b1b1ba230f:35019 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-27T13:40:41,578 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=29b1b1ba230f,39411,1732714839711 2024-11-27T13:40:41,585 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 29b1b1ba230f,39411,1732714839711, state=OPENING 2024-11-27T13:40:41,637 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-27T13:40:41,647 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39411-0x1017c8633370002, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T13:40:41,647 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35019-0x1017c8633370000, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T13:40:41,647 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1017c8633370001, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T13:40:41,647 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44709-0x1017c8633370003, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T13:40:41,649 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:41,649 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:41,649 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:41,649 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:41,651 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T13:40:41,654 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=29b1b1ba230f,39411,1732714839711}] 2024-11-27T13:40:41,837 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-27T13:40:41,840 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54017, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-27T13:40:41,854 INFO [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-27T13:40:41,854 INFO [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T13:40:41,855 INFO [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-27T13:40:41,858 INFO [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=29b1b1ba230f%2C39411%2C1732714839711.meta, suffix=.meta, logDir=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/WALs/29b1b1ba230f,39411,1732714839711, archiveDir=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/oldWALs, maxLogs=32 2024-11-27T13:40:41,874 DEBUG [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/WALs/29b1b1ba230f,39411,1732714839711/29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, exclude list is [], retry=0 2024-11-27T13:40:41,878 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38955,DS-fac1a99f-a8dd-4363-b07e-bd13cdba036b,DISK] 2024-11-27T13:40:41,878 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37007,DS-608d5eb3-8a33-4125-bd6d-2845739e66fc,DISK] 2024-11-27T13:40:41,878 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42991,DS-d60122c6-962a-4258-9fbb-6d2f24e1c6ac,DISK] 2024-11-27T13:40:41,881 INFO [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/WALs/29b1b1ba230f,39411,1732714839711/29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta 2024-11-27T13:40:41,881 DEBUG [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36243:36243),(127.0.0.1/127.0.0.1:36867:36867),(127.0.0.1/127.0.0.1:34247:34247)] 2024-11-27T13:40:41,881 DEBUG [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-27T13:40:41,883 DEBUG [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-27T13:40:41,885 DEBUG [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-27T13:40:41,889 INFO [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-27T13:40:41,893 DEBUG [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-27T13:40:41,894 DEBUG [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T13:40:41,894 DEBUG [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-27T13:40:41,894 DEBUG [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-27T13:40:41,897 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-27T13:40:41,899 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-27T13:40:41,899 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T13:40:41,900 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T13:40:41,900 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-27T13:40:41,901 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-27T13:40:41,902 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T13:40:41,902 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T13:40:41,903 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-27T13:40:41,904 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-27T13:40:41,904 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T13:40:41,905 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T13:40:41,905 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-27T13:40:41,906 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-27T13:40:41,906 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T13:40:41,907 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T13:40:41,907 DEBUG [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-27T13:40:41,909 DEBUG [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/hbase/meta/1588230740 2024-11-27T13:40:41,912 DEBUG [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/hbase/meta/1588230740 2024-11-27T13:40:41,914 DEBUG [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-27T13:40:41,914 DEBUG [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-27T13:40:41,915 DEBUG [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-27T13:40:41,918 DEBUG [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-27T13:40:41,919 INFO [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71970636, jitterRate=0.07244604825973511}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-27T13:40:41,920 DEBUG [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-27T13:40:41,921 DEBUG [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732714841894Writing region info on filesystem at 1732714841895 (+1 ms)Initializing all the Stores at 1732714841897 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732714841897Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732714841897Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732714841897Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732714841897Cleaning up temporary data from old regions at 1732714841914 (+17 ms)Running coprocessor post-open hooks at 1732714841920 (+6 ms)Region opened successfully at 1732714841920 2024-11-27T13:40:41,927 INFO [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732714841827 2024-11-27T13:40:41,939 DEBUG [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-27T13:40:41,940 INFO [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-27T13:40:41,942 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=29b1b1ba230f,39411,1732714839711 2024-11-27T13:40:41,945 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 29b1b1ba230f,39411,1732714839711, state=OPEN 2024-11-27T13:40:41,952 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44709-0x1017c8633370003, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:41,952 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35019-0x1017c8633370000, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:41,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39411-0x1017c8633370002, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:41,953 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:41,953 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:41,953 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:41,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1017c8633370001, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:41,953 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:41,954 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=29b1b1ba230f,39411,1732714839711 2024-11-27T13:40:41,960 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-27T13:40:41,960 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=29b1b1ba230f,39411,1732714839711 in 301 msec 2024-11-27T13:40:41,968 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-27T13:40:41,969 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 573 msec 2024-11-27T13:40:41,971 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-27T13:40:41,971 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-27T13:40:41,989 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T13:40:41,990 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=29b1b1ba230f,39411,1732714839711, seqNum=-1] 2024-11-27T13:40:42,013 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T13:40:42,015 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39321, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T13:40:42,034 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2510 sec 2024-11-27T13:40:42,034 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732714842034, completionTime=-1 2024-11-27T13:40:42,037 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-27T13:40:42,037 DEBUG [master/29b1b1ba230f:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T13:40:42,090 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-27T13:40:42,090 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732714902090 2024-11-27T13:40:42,090 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732714962090 2024-11-27T13:40:42,090 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 53 msec 2024-11-27T13:40:42,092 DEBUG [master/29b1b1ba230f:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-27T13:40:42,101 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=29b1b1ba230f,35019,1732714838827-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T13:40:42,102 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=29b1b1ba230f,35019,1732714838827-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T13:40:42,102 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=29b1b1ba230f,35019,1732714838827-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T13:40:42,104 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-29b1b1ba230f:35019, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T13:40:42,105 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-27T13:40:42,110 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-27T13:40:42,114 DEBUG [master/29b1b1ba230f:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-27T13:40:42,139 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.268sec 2024-11-27T13:40:42,141 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-27T13:40:42,142 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-27T13:40:42,143 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-27T13:40:42,143 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-27T13:40:42,143 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-27T13:40:42,144 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=29b1b1ba230f,35019,1732714838827-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-27T13:40:42,145 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=29b1b1ba230f,35019,1732714838827-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-27T13:40:42,149 DEBUG [master/29b1b1ba230f:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-27T13:40:42,150 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-27T13:40:42,150 INFO [master/29b1b1ba230f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=29b1b1ba230f,35019,1732714838827-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T13:40:42,211 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b48c89d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T13:40:42,215 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-27T13:40:42,215 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-27T13:40:42,218 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 29b1b1ba230f,35019,-1 for getting cluster id 2024-11-27T13:40:42,220 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T13:40:42,228 DEBUG [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80a0a673-acef-4b6a-956f-a08e628106dc' 2024-11-27T13:40:42,229 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T13:40:42,230 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80a0a673-acef-4b6a-956f-a08e628106dc" 2024-11-27T13:40:42,230 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f872de9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T13:40:42,230 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [29b1b1ba230f,35019,-1] 2024-11-27T13:40:42,232 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T13:40:42,234 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T13:40:42,235 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58926, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T13:40:42,238 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10d42a85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T13:40:42,238 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T13:40:42,245 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=29b1b1ba230f,39411,1732714839711, seqNum=-1] 2024-11-27T13:40:42,246 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T13:40:42,249 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33952, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T13:40:42,267 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=29b1b1ba230f,35019,1732714838827 2024-11-27T13:40:42,271 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T13:40:42,275 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 29b1b1ba230f,35019,1732714838827 2024-11-27T13:40:42,278 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@71305feb 2024-11-27T13:40:42,279 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T13:40:42,281 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58928, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-27T13:40:42,288 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35019 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'test', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T13:40:42,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35019 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test 2024-11-27T13:40:42,300 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T13:40:42,303 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35019 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "test" procId is: 4 2024-11-27T13:40:42,303 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T13:40:42,307 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T13:40:42,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T13:40:42,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37007 is added to blk_1073741837_1013 (size=358) 2024-11-27T13:40:42,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38955 is added to blk_1073741837_1013 (size=358) 2024-11-27T13:40:42,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42991 is added to blk_1073741837_1013 (size=358) 2024-11-27T13:40:42,325 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 51060408ec4ea38afe378df89549aeae, NAME => 'test,,1732714842283.51060408ec4ea38afe378df89549aeae.', STARTKEY => '', ENDKEY => 'bbb'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237 2024-11-27T13:40:42,326 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 2b6b183b9bc56b2bc4dc6f8968ee01c8, NAME => 'test,bbb,1732714842283.2b6b183b9bc56b2bc4dc6f8968ee01c8.', STARTKEY => 'bbb', ENDKEY => 'ccc'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237 2024-11-27T13:40:42,326 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 0aef6a7e4e138b1b9d7f4aea6971acc6, NAME => 'test,ccc,1732714842283.0aef6a7e4e138b1b9d7f4aea6971acc6.', STARTKEY => 'ccc', ENDKEY => 'ddd'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237 2024-11-27T13:40:42,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38955 is added to blk_1073741839_1015 (size=42) 2024-11-27T13:40:42,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37007 is added to blk_1073741839_1015 (size=42) 2024-11-27T13:40:42,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42991 is added to blk_1073741839_1015 (size=42) 2024-11-27T13:40:42,343 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,,1732714842283.51060408ec4ea38afe378df89549aeae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T13:40:42,344 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 51060408ec4ea38afe378df89549aeae, disabling compactions & flushes 2024-11-27T13:40:42,344 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,,1732714842283.51060408ec4ea38afe378df89549aeae. 2024-11-27T13:40:42,344 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1732714842283.51060408ec4ea38afe378df89549aeae. 2024-11-27T13:40:42,344 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,,1732714842283.51060408ec4ea38afe378df89549aeae. after waiting 0 ms 2024-11-27T13:40:42,344 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,,1732714842283.51060408ec4ea38afe378df89549aeae. 2024-11-27T13:40:42,344 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,,1732714842283.51060408ec4ea38afe378df89549aeae. 2024-11-27T13:40:42,344 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 51060408ec4ea38afe378df89549aeae: Waiting for close lock at 1732714842344Disabling compacts and flushes for region at 1732714842344Disabling writes for close at 1732714842344Writing region close event to WAL at 1732714842344Closed at 1732714842344 2024-11-27T13:40:42,344 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 49523b5c318014c3e33791ea6f5e308d, NAME => 'test,ddd,1732714842283.49523b5c318014c3e33791ea6f5e308d.', STARTKEY => 'ddd', ENDKEY => 'eee'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237 2024-11-27T13:40:42,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42991 is added to blk_1073741838_1014 (size=45) 2024-11-27T13:40:42,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37007 is added to blk_1073741838_1014 (size=45) 2024-11-27T13:40:42,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38955 is added to blk_1073741838_1014 (size=45) 2024-11-27T13:40:42,355 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ccc,1732714842283.0aef6a7e4e138b1b9d7f4aea6971acc6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T13:40:42,355 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 0aef6a7e4e138b1b9d7f4aea6971acc6, disabling compactions & flushes 2024-11-27T13:40:42,355 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ccc,1732714842283.0aef6a7e4e138b1b9d7f4aea6971acc6. 2024-11-27T13:40:42,355 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1732714842283.0aef6a7e4e138b1b9d7f4aea6971acc6. 2024-11-27T13:40:42,355 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1732714842283.0aef6a7e4e138b1b9d7f4aea6971acc6. after waiting 0 ms 2024-11-27T13:40:42,355 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1732714842283.0aef6a7e4e138b1b9d7f4aea6971acc6. 2024-11-27T13:40:42,355 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ccc,1732714842283.0aef6a7e4e138b1b9d7f4aea6971acc6. 2024-11-27T13:40:42,355 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 0aef6a7e4e138b1b9d7f4aea6971acc6: Waiting for close lock at 1732714842355Disabling compacts and flushes for region at 1732714842355Disabling writes for close at 1732714842355Writing region close event to WAL at 1732714842355Closed at 1732714842355 2024-11-27T13:40:42,356 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 4d1ad27263331aa341edfa82a041de56, NAME => 'test,eee,1732714842283.4d1ad27263331aa341edfa82a041de56.', STARTKEY => 'eee', ENDKEY => 'fff'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237 2024-11-27T13:40:42,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42991 is added to blk_1073741840_1016 (size=45) 2024-11-27T13:40:42,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38955 is added to blk_1073741840_1016 (size=45) 2024-11-27T13:40:42,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37007 is added to blk_1073741840_1016 (size=45) 2024-11-27T13:40:42,367 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,bbb,1732714842283.2b6b183b9bc56b2bc4dc6f8968ee01c8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T13:40:42,368 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 2b6b183b9bc56b2bc4dc6f8968ee01c8, disabling compactions & flushes 2024-11-27T13:40:42,368 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,bbb,1732714842283.2b6b183b9bc56b2bc4dc6f8968ee01c8. 2024-11-27T13:40:42,368 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1732714842283.2b6b183b9bc56b2bc4dc6f8968ee01c8. 2024-11-27T13:40:42,368 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1732714842283.2b6b183b9bc56b2bc4dc6f8968ee01c8. after waiting 0 ms 2024-11-27T13:40:42,368 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1732714842283.2b6b183b9bc56b2bc4dc6f8968ee01c8. 2024-11-27T13:40:42,368 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,bbb,1732714842283.2b6b183b9bc56b2bc4dc6f8968ee01c8. 2024-11-27T13:40:42,368 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 2b6b183b9bc56b2bc4dc6f8968ee01c8: Waiting for close lock at 1732714842368Disabling compacts and flushes for region at 1732714842368Disabling writes for close at 1732714842368Writing region close event to WAL at 1732714842368Closed at 1732714842368 2024-11-27T13:40:42,369 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 8fc6390c5b3b9356bc4885792f770cf2, NAME => 'test,fff,1732714842283.8fc6390c5b3b9356bc4885792f770cf2.', STARTKEY => 'fff', ENDKEY => 'ggg'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237 2024-11-27T13:40:42,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37007 is added to blk_1073741842_1018 (size=45) 2024-11-27T13:40:42,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42991 is added to blk_1073741842_1018 (size=45) 2024-11-27T13:40:42,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38955 is added to blk_1073741842_1018 (size=45) 2024-11-27T13:40:42,384 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,eee,1732714842283.4d1ad27263331aa341edfa82a041de56.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T13:40:42,384 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 4d1ad27263331aa341edfa82a041de56, disabling compactions & flushes 2024-11-27T13:40:42,384 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,eee,1732714842283.4d1ad27263331aa341edfa82a041de56. 2024-11-27T13:40:42,384 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1732714842283.4d1ad27263331aa341edfa82a041de56. 2024-11-27T13:40:42,384 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,eee,1732714842283.4d1ad27263331aa341edfa82a041de56. after waiting 0 ms 2024-11-27T13:40:42,384 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,eee,1732714842283.4d1ad27263331aa341edfa82a041de56. 2024-11-27T13:40:42,385 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,eee,1732714842283.4d1ad27263331aa341edfa82a041de56. 2024-11-27T13:40:42,385 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 4d1ad27263331aa341edfa82a041de56: Waiting for close lock at 1732714842384Disabling compacts and flushes for region at 1732714842384Disabling writes for close at 1732714842384Writing region close event to WAL at 1732714842384Closed at 1732714842384 2024-11-27T13:40:42,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38955 is added to blk_1073741841_1017 (size=45) 2024-11-27T13:40:42,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37007 is added to blk_1073741841_1017 (size=45) 2024-11-27T13:40:42,386 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 85dc78011fc464d1446075ad14a217de, NAME => 'test,ggg,1732714842283.85dc78011fc464d1446075ad14a217de.', STARTKEY => 'ggg', ENDKEY => 'hhh'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237 2024-11-27T13:40:42,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42991 is added to blk_1073741841_1017 (size=45) 2024-11-27T13:40:42,388 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ddd,1732714842283.49523b5c318014c3e33791ea6f5e308d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T13:40:42,388 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 49523b5c318014c3e33791ea6f5e308d, disabling compactions & flushes 2024-11-27T13:40:42,388 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ddd,1732714842283.49523b5c318014c3e33791ea6f5e308d. 2024-11-27T13:40:42,388 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1732714842283.49523b5c318014c3e33791ea6f5e308d. 2024-11-27T13:40:42,388 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1732714842283.49523b5c318014c3e33791ea6f5e308d. after waiting 0 ms 2024-11-27T13:40:42,388 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1732714842283.49523b5c318014c3e33791ea6f5e308d. 2024-11-27T13:40:42,388 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ddd,1732714842283.49523b5c318014c3e33791ea6f5e308d. 2024-11-27T13:40:42,388 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 49523b5c318014c3e33791ea6f5e308d: Waiting for close lock at 1732714842388Disabling compacts and flushes for region at 1732714842388Disabling writes for close at 1732714842388Writing region close event to WAL at 1732714842388Closed at 1732714842388 2024-11-27T13:40:42,389 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9ee34ecbedcb7dfdabcb99a67ac21045, NAME => 'test,hhh,1732714842283.9ee34ecbedcb7dfdabcb99a67ac21045.', STARTKEY => 'hhh', ENDKEY => 'iii'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237 2024-11-27T13:40:42,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38955 is added to blk_1073741843_1019 (size=45) 2024-11-27T13:40:42,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42991 is added to blk_1073741843_1019 (size=45) 2024-11-27T13:40:42,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37007 is added to blk_1073741843_1019 (size=45) 2024-11-27T13:40:42,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37007 is added to blk_1073741845_1021 (size=45) 2024-11-27T13:40:42,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T13:40:42,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38955 is added to blk_1073741845_1021 (size=45) 2024-11-27T13:40:42,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42991 is added to blk_1073741845_1021 (size=45) 2024-11-27T13:40:42,415 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,hhh,1732714842283.9ee34ecbedcb7dfdabcb99a67ac21045.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T13:40:42,415 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 9ee34ecbedcb7dfdabcb99a67ac21045, disabling compactions & flushes 2024-11-27T13:40:42,415 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,hhh,1732714842283.9ee34ecbedcb7dfdabcb99a67ac21045. 2024-11-27T13:40:42,415 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1732714842283.9ee34ecbedcb7dfdabcb99a67ac21045. 2024-11-27T13:40:42,416 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1732714842283.9ee34ecbedcb7dfdabcb99a67ac21045. after waiting 0 ms 2024-11-27T13:40:42,416 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1732714842283.9ee34ecbedcb7dfdabcb99a67ac21045. 2024-11-27T13:40:42,416 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,hhh,1732714842283.9ee34ecbedcb7dfdabcb99a67ac21045. 2024-11-27T13:40:42,416 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9ee34ecbedcb7dfdabcb99a67ac21045: Waiting for close lock at 1732714842415Disabling compacts and flushes for region at 1732714842415Disabling writes for close at 1732714842416 (+1 ms)Writing region close event to WAL at 1732714842416Closed at 1732714842416 2024-11-27T13:40:42,416 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => dcf91d39846df82f849110c861f2a7d0, NAME => 'test,iii,1732714842283.dcf91d39846df82f849110c861f2a7d0.', STARTKEY => 'iii', ENDKEY => 'jjj'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237 2024-11-27T13:40:42,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38955 is added to blk_1073741844_1020 (size=45) 2024-11-27T13:40:42,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37007 is added to blk_1073741844_1020 (size=45) 2024-11-27T13:40:42,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42991 is added to blk_1073741844_1020 (size=45) 2024-11-27T13:40:42,421 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ggg,1732714842283.85dc78011fc464d1446075ad14a217de.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T13:40:42,421 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 85dc78011fc464d1446075ad14a217de, disabling compactions & flushes 2024-11-27T13:40:42,421 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ggg,1732714842283.85dc78011fc464d1446075ad14a217de. 2024-11-27T13:40:42,421 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1732714842283.85dc78011fc464d1446075ad14a217de. 2024-11-27T13:40:42,421 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1732714842283.85dc78011fc464d1446075ad14a217de. after waiting 0 ms 2024-11-27T13:40:42,421 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1732714842283.85dc78011fc464d1446075ad14a217de. 2024-11-27T13:40:42,421 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ggg,1732714842283.85dc78011fc464d1446075ad14a217de. 2024-11-27T13:40:42,421 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 85dc78011fc464d1446075ad14a217de: Waiting for close lock at 1732714842421Disabling compacts and flushes for region at 1732714842421Disabling writes for close at 1732714842421Writing region close event to WAL at 1732714842421Closed at 1732714842421 2024-11-27T13:40:42,422 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => e7a7c9c02e40375cd974ffe51b6da75b, NAME => 'test,jjj,1732714842283.e7a7c9c02e40375cd974ffe51b6da75b.', STARTKEY => 'jjj', ENDKEY => 'kkk'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237 2024-11-27T13:40:42,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37007 is added to blk_1073741846_1022 (size=45) 2024-11-27T13:40:42,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42991 is added to blk_1073741846_1022 (size=45) 2024-11-27T13:40:42,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38955 is added to blk_1073741846_1022 (size=45) 2024-11-27T13:40:42,434 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,iii,1732714842283.dcf91d39846df82f849110c861f2a7d0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T13:40:42,434 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing dcf91d39846df82f849110c861f2a7d0, disabling compactions & flushes 2024-11-27T13:40:42,434 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,iii,1732714842283.dcf91d39846df82f849110c861f2a7d0. 2024-11-27T13:40:42,434 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1732714842283.dcf91d39846df82f849110c861f2a7d0. 2024-11-27T13:40:42,434 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,iii,1732714842283.dcf91d39846df82f849110c861f2a7d0. after waiting 0 ms 2024-11-27T13:40:42,435 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,iii,1732714842283.dcf91d39846df82f849110c861f2a7d0. 2024-11-27T13:40:42,435 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,iii,1732714842283.dcf91d39846df82f849110c861f2a7d0. 2024-11-27T13:40:42,435 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for dcf91d39846df82f849110c861f2a7d0: Waiting for close lock at 1732714842434Disabling compacts and flushes for region at 1732714842434Disabling writes for close at 1732714842435 (+1 ms)Writing region close event to WAL at 1732714842435Closed at 1732714842435 2024-11-27T13:40:42,435 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => eeadb5717c90d7f7a33c12d4883bf2b0, NAME => 'test,kkk,1732714842283.eeadb5717c90d7f7a33c12d4883bf2b0.', STARTKEY => 'kkk', ENDKEY => 'lll'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237 2024-11-27T13:40:42,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42991 is added to blk_1073741847_1023 (size=45) 2024-11-27T13:40:42,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38955 is added to blk_1073741847_1023 (size=45) 2024-11-27T13:40:42,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37007 is added to blk_1073741847_1023 (size=45) 2024-11-27T13:40:42,444 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,jjj,1732714842283.e7a7c9c02e40375cd974ffe51b6da75b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T13:40:42,445 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing e7a7c9c02e40375cd974ffe51b6da75b, disabling compactions & flushes 2024-11-27T13:40:42,445 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,jjj,1732714842283.e7a7c9c02e40375cd974ffe51b6da75b. 2024-11-27T13:40:42,445 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1732714842283.e7a7c9c02e40375cd974ffe51b6da75b. 2024-11-27T13:40:42,445 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1732714842283.e7a7c9c02e40375cd974ffe51b6da75b. after waiting 0 ms 2024-11-27T13:40:42,445 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1732714842283.e7a7c9c02e40375cd974ffe51b6da75b. 2024-11-27T13:40:42,445 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,jjj,1732714842283.e7a7c9c02e40375cd974ffe51b6da75b. 2024-11-27T13:40:42,445 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for e7a7c9c02e40375cd974ffe51b6da75b: Waiting for close lock at 1732714842445Disabling compacts and flushes for region at 1732714842445Disabling writes for close at 1732714842445Writing region close event to WAL at 1732714842445Closed at 1732714842445 2024-11-27T13:40:42,446 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => a9a679b7eb3754995869ddbc90163b1c, NAME => 'test,lll,1732714842283.a9a679b7eb3754995869ddbc90163b1c.', STARTKEY => 'lll', ENDKEY => 'mmm'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237 2024-11-27T13:40:42,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37007 is added to blk_1073741848_1024 (size=45) 2024-11-27T13:40:42,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42991 is added to blk_1073741848_1024 (size=45) 2024-11-27T13:40:42,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38955 is added to blk_1073741848_1024 (size=45) 2024-11-27T13:40:42,452 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,kkk,1732714842283.eeadb5717c90d7f7a33c12d4883bf2b0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T13:40:42,453 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing eeadb5717c90d7f7a33c12d4883bf2b0, disabling compactions & flushes 2024-11-27T13:40:42,453 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,kkk,1732714842283.eeadb5717c90d7f7a33c12d4883bf2b0. 2024-11-27T13:40:42,453 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1732714842283.eeadb5717c90d7f7a33c12d4883bf2b0. 2024-11-27T13:40:42,453 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1732714842283.eeadb5717c90d7f7a33c12d4883bf2b0. after waiting 0 ms 2024-11-27T13:40:42,453 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1732714842283.eeadb5717c90d7f7a33c12d4883bf2b0. 2024-11-27T13:40:42,453 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,kkk,1732714842283.eeadb5717c90d7f7a33c12d4883bf2b0. 2024-11-27T13:40:42,453 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for eeadb5717c90d7f7a33c12d4883bf2b0: Waiting for close lock at 1732714842453Disabling compacts and flushes for region at 1732714842453Disabling writes for close at 1732714842453Writing region close event to WAL at 1732714842453Closed at 1732714842453 2024-11-27T13:40:42,454 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => cccf7ce6a567350b0ae162454b0a9379, NAME => 'test,mmm,1732714842283.cccf7ce6a567350b0ae162454b0a9379.', STARTKEY => 'mmm', ENDKEY => 'nnn'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237 2024-11-27T13:40:42,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37007 is added to blk_1073741849_1025 (size=45) 2024-11-27T13:40:42,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42991 is added to blk_1073741849_1025 (size=45) 2024-11-27T13:40:42,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38955 is added to blk_1073741849_1025 (size=45) 2024-11-27T13:40:42,463 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,lll,1732714842283.a9a679b7eb3754995869ddbc90163b1c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T13:40:42,463 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing a9a679b7eb3754995869ddbc90163b1c, disabling compactions & flushes 2024-11-27T13:40:42,463 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,lll,1732714842283.a9a679b7eb3754995869ddbc90163b1c. 2024-11-27T13:40:42,463 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1732714842283.a9a679b7eb3754995869ddbc90163b1c. 2024-11-27T13:40:42,463 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,lll,1732714842283.a9a679b7eb3754995869ddbc90163b1c. after waiting 0 ms 2024-11-27T13:40:42,463 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,lll,1732714842283.a9a679b7eb3754995869ddbc90163b1c. 2024-11-27T13:40:42,463 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,lll,1732714842283.a9a679b7eb3754995869ddbc90163b1c. 2024-11-27T13:40:42,464 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for a9a679b7eb3754995869ddbc90163b1c: Waiting for close lock at 1732714842463Disabling compacts and flushes for region at 1732714842463Disabling writes for close at 1732714842463Writing region close event to WAL at 1732714842463Closed at 1732714842463 2024-11-27T13:40:42,464 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => d095bb01efaabc5bf5541d373d5419fa, NAME => 'test,nnn,1732714842283.d095bb01efaabc5bf5541d373d5419fa.', STARTKEY => 'nnn', ENDKEY => 'ooo'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237 2024-11-27T13:40:42,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37007 is added to blk_1073741850_1026 (size=45) 2024-11-27T13:40:42,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38955 is added to blk_1073741850_1026 (size=45) 2024-11-27T13:40:42,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42991 is added to blk_1073741850_1026 (size=45) 2024-11-27T13:40:42,470 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,mmm,1732714842283.cccf7ce6a567350b0ae162454b0a9379.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T13:40:42,470 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing cccf7ce6a567350b0ae162454b0a9379, disabling compactions & flushes 2024-11-27T13:40:42,470 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,mmm,1732714842283.cccf7ce6a567350b0ae162454b0a9379. 2024-11-27T13:40:42,470 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1732714842283.cccf7ce6a567350b0ae162454b0a9379. 2024-11-27T13:40:42,470 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1732714842283.cccf7ce6a567350b0ae162454b0a9379. after waiting 0 ms 2024-11-27T13:40:42,471 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1732714842283.cccf7ce6a567350b0ae162454b0a9379. 2024-11-27T13:40:42,471 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,mmm,1732714842283.cccf7ce6a567350b0ae162454b0a9379. 2024-11-27T13:40:42,471 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for cccf7ce6a567350b0ae162454b0a9379: Waiting for close lock at 1732714842470Disabling compacts and flushes for region at 1732714842470Disabling writes for close at 1732714842471 (+1 ms)Writing region close event to WAL at 1732714842471Closed at 1732714842471 2024-11-27T13:40:42,471 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9c46624524e53a899138ac5a715a35d6, NAME => 'test,ooo,1732714842283.9c46624524e53a899138ac5a715a35d6.', STARTKEY => 'ooo', ENDKEY => 'ppp'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237 2024-11-27T13:40:42,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38955 is added to blk_1073741851_1027 (size=45) 2024-11-27T13:40:42,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37007 is added to blk_1073741851_1027 (size=45) 2024-11-27T13:40:42,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42991 is added to blk_1073741851_1027 (size=45) 2024-11-27T13:40:42,480 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,nnn,1732714842283.d095bb01efaabc5bf5541d373d5419fa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T13:40:42,480 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing d095bb01efaabc5bf5541d373d5419fa, disabling compactions & flushes 2024-11-27T13:40:42,480 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,nnn,1732714842283.d095bb01efaabc5bf5541d373d5419fa. 2024-11-27T13:40:42,480 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1732714842283.d095bb01efaabc5bf5541d373d5419fa. 2024-11-27T13:40:42,480 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1732714842283.d095bb01efaabc5bf5541d373d5419fa. after waiting 0 ms 2024-11-27T13:40:42,480 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1732714842283.d095bb01efaabc5bf5541d373d5419fa. 2024-11-27T13:40:42,480 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,nnn,1732714842283.d095bb01efaabc5bf5541d373d5419fa. 2024-11-27T13:40:42,480 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for d095bb01efaabc5bf5541d373d5419fa: Waiting for close lock at 1732714842480Disabling compacts and flushes for region at 1732714842480Disabling writes for close at 1732714842480Writing region close event to WAL at 1732714842480Closed at 1732714842480 2024-11-27T13:40:42,481 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => de4deb0c16ab89c9c56fc8912227f1c5, NAME => 'test,ppp,1732714842283.de4deb0c16ab89c9c56fc8912227f1c5.', STARTKEY => 'ppp', ENDKEY => 'qqq'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237 2024-11-27T13:40:42,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42991 is added to blk_1073741852_1028 (size=45) 2024-11-27T13:40:42,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38955 is added to blk_1073741852_1028 (size=45) 2024-11-27T13:40:42,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37007 is added to blk_1073741852_1028 (size=45) 2024-11-27T13:40:42,489 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ooo,1732714842283.9c46624524e53a899138ac5a715a35d6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T13:40:42,490 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 9c46624524e53a899138ac5a715a35d6, disabling compactions & flushes 2024-11-27T13:40:42,490 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ooo,1732714842283.9c46624524e53a899138ac5a715a35d6. 2024-11-27T13:40:42,490 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1732714842283.9c46624524e53a899138ac5a715a35d6. 2024-11-27T13:40:42,490 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1732714842283.9c46624524e53a899138ac5a715a35d6. after waiting 0 ms 2024-11-27T13:40:42,490 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1732714842283.9c46624524e53a899138ac5a715a35d6. 2024-11-27T13:40:42,490 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ooo,1732714842283.9c46624524e53a899138ac5a715a35d6. 2024-11-27T13:40:42,490 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9c46624524e53a899138ac5a715a35d6: Waiting for close lock at 1732714842490Disabling compacts and flushes for region at 1732714842490Disabling writes for close at 1732714842490Writing region close event to WAL at 1732714842490Closed at 1732714842490 2024-11-27T13:40:42,491 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6b448045aa14a354ef4bc74b08978138, NAME => 'test,qqq,1732714842283.6b448045aa14a354ef4bc74b08978138.', STARTKEY => 'qqq', ENDKEY => 'rrr'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237 2024-11-27T13:40:42,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37007 is added to blk_1073741853_1029 (size=45) 2024-11-27T13:40:42,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42991 is added to blk_1073741853_1029 (size=45) 2024-11-27T13:40:42,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38955 is added to blk_1073741853_1029 (size=45) 2024-11-27T13:40:42,497 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ppp,1732714842283.de4deb0c16ab89c9c56fc8912227f1c5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T13:40:42,497 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing de4deb0c16ab89c9c56fc8912227f1c5, disabling compactions & flushes 2024-11-27T13:40:42,498 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ppp,1732714842283.de4deb0c16ab89c9c56fc8912227f1c5. 2024-11-27T13:40:42,498 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1732714842283.de4deb0c16ab89c9c56fc8912227f1c5. 2024-11-27T13:40:42,498 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1732714842283.de4deb0c16ab89c9c56fc8912227f1c5. after waiting 0 ms 2024-11-27T13:40:42,498 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1732714842283.de4deb0c16ab89c9c56fc8912227f1c5. 2024-11-27T13:40:42,498 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ppp,1732714842283.de4deb0c16ab89c9c56fc8912227f1c5. 2024-11-27T13:40:42,498 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for de4deb0c16ab89c9c56fc8912227f1c5: Waiting for close lock at 1732714842497Disabling compacts and flushes for region at 1732714842497Disabling writes for close at 1732714842498 (+1 ms)Writing region close event to WAL at 1732714842498Closed at 1732714842498 2024-11-27T13:40:42,499 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 8700ed52628a2e71e47ec460999f2612, NAME => 'test,rrr,1732714842283.8700ed52628a2e71e47ec460999f2612.', STARTKEY => 'rrr', ENDKEY => 'sss'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237 2024-11-27T13:40:42,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42991 is added to blk_1073741854_1030 (size=45) 2024-11-27T13:40:42,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38955 is added to blk_1073741854_1030 (size=45) 2024-11-27T13:40:42,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37007 is added to blk_1073741854_1030 (size=45) 2024-11-27T13:40:42,514 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,qqq,1732714842283.6b448045aa14a354ef4bc74b08978138.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T13:40:42,514 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 6b448045aa14a354ef4bc74b08978138, disabling compactions & flushes 2024-11-27T13:40:42,514 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,qqq,1732714842283.6b448045aa14a354ef4bc74b08978138. 2024-11-27T13:40:42,514 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1732714842283.6b448045aa14a354ef4bc74b08978138. 2024-11-27T13:40:42,514 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1732714842283.6b448045aa14a354ef4bc74b08978138. after waiting 0 ms 2024-11-27T13:40:42,514 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1732714842283.6b448045aa14a354ef4bc74b08978138. 2024-11-27T13:40:42,514 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,qqq,1732714842283.6b448045aa14a354ef4bc74b08978138. 2024-11-27T13:40:42,514 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6b448045aa14a354ef4bc74b08978138: Waiting for close lock at 1732714842514Disabling compacts and flushes for region at 1732714842514Disabling writes for close at 1732714842514Writing region close event to WAL at 1732714842514Closed at 1732714842514 2024-11-27T13:40:42,515 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f5f0e25bfe9d22339132aab7b2f16f09, NAME => 'test,sss,1732714842283.f5f0e25bfe9d22339132aab7b2f16f09.', STARTKEY => 'sss', ENDKEY => 'ttt'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237 2024-11-27T13:40:42,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38955 is added to blk_1073741855_1031 (size=45) 2024-11-27T13:40:42,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42991 is added to blk_1073741855_1031 (size=45) 2024-11-27T13:40:42,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37007 is added to blk_1073741855_1031 (size=45) 2024-11-27T13:40:42,523 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,rrr,1732714842283.8700ed52628a2e71e47ec460999f2612.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T13:40:42,523 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 8700ed52628a2e71e47ec460999f2612, disabling compactions & flushes 2024-11-27T13:40:42,523 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,rrr,1732714842283.8700ed52628a2e71e47ec460999f2612. 2024-11-27T13:40:42,523 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1732714842283.8700ed52628a2e71e47ec460999f2612. 2024-11-27T13:40:42,523 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1732714842283.8700ed52628a2e71e47ec460999f2612. after waiting 0 ms 2024-11-27T13:40:42,523 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1732714842283.8700ed52628a2e71e47ec460999f2612. 2024-11-27T13:40:42,523 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,rrr,1732714842283.8700ed52628a2e71e47ec460999f2612. 2024-11-27T13:40:42,523 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 8700ed52628a2e71e47ec460999f2612: Waiting for close lock at 1732714842523Disabling compacts and flushes for region at 1732714842523Disabling writes for close at 1732714842523Writing region close event to WAL at 1732714842523Closed at 1732714842523 2024-11-27T13:40:42,524 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 922622f55d3d471d091912fe46d7c8d8, NAME => 'test,ttt,1732714842283.922622f55d3d471d091912fe46d7c8d8.', STARTKEY => 'ttt', ENDKEY => 'uuu'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237 2024-11-27T13:40:42,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38955 is added to blk_1073741856_1032 (size=45) 2024-11-27T13:40:42,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42991 is added to blk_1073741856_1032 (size=45) 2024-11-27T13:40:42,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37007 is added to blk_1073741856_1032 (size=45) 2024-11-27T13:40:42,533 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,sss,1732714842283.f5f0e25bfe9d22339132aab7b2f16f09.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T13:40:42,533 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing f5f0e25bfe9d22339132aab7b2f16f09, disabling compactions & flushes 2024-11-27T13:40:42,533 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,sss,1732714842283.f5f0e25bfe9d22339132aab7b2f16f09. 2024-11-27T13:40:42,533 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1732714842283.f5f0e25bfe9d22339132aab7b2f16f09. 2024-11-27T13:40:42,534 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,sss,1732714842283.f5f0e25bfe9d22339132aab7b2f16f09. after waiting 0 ms 2024-11-27T13:40:42,534 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,sss,1732714842283.f5f0e25bfe9d22339132aab7b2f16f09. 2024-11-27T13:40:42,534 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,sss,1732714842283.f5f0e25bfe9d22339132aab7b2f16f09. 2024-11-27T13:40:42,534 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for f5f0e25bfe9d22339132aab7b2f16f09: Waiting for close lock at 1732714842533Disabling compacts and flushes for region at 1732714842533Disabling writes for close at 1732714842534 (+1 ms)Writing region close event to WAL at 1732714842534Closed at 1732714842534 2024-11-27T13:40:42,535 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 457728e6ba667138202e8a0dfa04b5b7, NAME => 'test,uuu,1732714842283.457728e6ba667138202e8a0dfa04b5b7.', STARTKEY => 'uuu', ENDKEY => 'vvv'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237 2024-11-27T13:40:42,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38955 is added to blk_1073741857_1033 (size=45) 2024-11-27T13:40:42,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37007 is added to blk_1073741857_1033 (size=45) 2024-11-27T13:40:42,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42991 is added to blk_1073741857_1033 (size=45) 2024-11-27T13:40:42,540 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ttt,1732714842283.922622f55d3d471d091912fe46d7c8d8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T13:40:42,540 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 922622f55d3d471d091912fe46d7c8d8, disabling compactions & flushes 2024-11-27T13:40:42,540 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ttt,1732714842283.922622f55d3d471d091912fe46d7c8d8. 2024-11-27T13:40:42,541 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1732714842283.922622f55d3d471d091912fe46d7c8d8. 2024-11-27T13:40:42,541 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1732714842283.922622f55d3d471d091912fe46d7c8d8. after waiting 0 ms 2024-11-27T13:40:42,541 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1732714842283.922622f55d3d471d091912fe46d7c8d8. 2024-11-27T13:40:42,541 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ttt,1732714842283.922622f55d3d471d091912fe46d7c8d8. 2024-11-27T13:40:42,541 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 922622f55d3d471d091912fe46d7c8d8: Waiting for close lock at 1732714842540Disabling compacts and flushes for region at 1732714842540Disabling writes for close at 1732714842541 (+1 ms)Writing region close event to WAL at 1732714842541Closed at 1732714842541 2024-11-27T13:40:42,541 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => c4f747499d4a9438c3cddbfb1ead0d2b, NAME => 'test,vvv,1732714842283.c4f747499d4a9438c3cddbfb1ead0d2b.', STARTKEY => 'vvv', ENDKEY => 'www'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237 2024-11-27T13:40:42,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42991 is added to blk_1073741858_1034 (size=45) 2024-11-27T13:40:42,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37007 is added to blk_1073741858_1034 (size=45) 2024-11-27T13:40:42,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38955 is added to blk_1073741858_1034 (size=45) 2024-11-27T13:40:42,550 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,uuu,1732714842283.457728e6ba667138202e8a0dfa04b5b7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T13:40:42,550 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 457728e6ba667138202e8a0dfa04b5b7, disabling compactions & flushes 2024-11-27T13:40:42,550 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,uuu,1732714842283.457728e6ba667138202e8a0dfa04b5b7. 2024-11-27T13:40:42,550 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1732714842283.457728e6ba667138202e8a0dfa04b5b7. 2024-11-27T13:40:42,551 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1732714842283.457728e6ba667138202e8a0dfa04b5b7. after waiting 0 ms 2024-11-27T13:40:42,551 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1732714842283.457728e6ba667138202e8a0dfa04b5b7. 2024-11-27T13:40:42,551 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,uuu,1732714842283.457728e6ba667138202e8a0dfa04b5b7. 2024-11-27T13:40:42,551 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 457728e6ba667138202e8a0dfa04b5b7: Waiting for close lock at 1732714842550Disabling compacts and flushes for region at 1732714842550Disabling writes for close at 1732714842551 (+1 ms)Writing region close event to WAL at 1732714842551Closed at 1732714842551 2024-11-27T13:40:42,552 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ed774066130adf09502c61d07d72f5ab, NAME => 'test,www,1732714842283.ed774066130adf09502c61d07d72f5ab.', STARTKEY => 'www', ENDKEY => 'xxx'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237 2024-11-27T13:40:42,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42991 is added to blk_1073741859_1035 (size=45) 2024-11-27T13:40:42,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38955 is added to blk_1073741859_1035 (size=45) 2024-11-27T13:40:42,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37007 is added to blk_1073741859_1035 (size=45) 2024-11-27T13:40:42,556 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,vvv,1732714842283.c4f747499d4a9438c3cddbfb1ead0d2b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T13:40:42,557 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing c4f747499d4a9438c3cddbfb1ead0d2b, disabling compactions & flushes 2024-11-27T13:40:42,557 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,vvv,1732714842283.c4f747499d4a9438c3cddbfb1ead0d2b. 2024-11-27T13:40:42,557 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1732714842283.c4f747499d4a9438c3cddbfb1ead0d2b. 2024-11-27T13:40:42,557 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1732714842283.c4f747499d4a9438c3cddbfb1ead0d2b. after waiting 0 ms 2024-11-27T13:40:42,557 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1732714842283.c4f747499d4a9438c3cddbfb1ead0d2b. 2024-11-27T13:40:42,557 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,vvv,1732714842283.c4f747499d4a9438c3cddbfb1ead0d2b. 2024-11-27T13:40:42,557 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for c4f747499d4a9438c3cddbfb1ead0d2b: Waiting for close lock at 1732714842557Disabling compacts and flushes for region at 1732714842557Disabling writes for close at 1732714842557Writing region close event to WAL at 1732714842557Closed at 1732714842557 2024-11-27T13:40:42,558 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 5c6bad09b1898b3a3810c7bfeb938da4, NAME => 'test,xxx,1732714842283.5c6bad09b1898b3a3810c7bfeb938da4.', STARTKEY => 'xxx', ENDKEY => 'yyy'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237 2024-11-27T13:40:42,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37007 is added to blk_1073741860_1036 (size=45) 2024-11-27T13:40:42,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38955 is added to blk_1073741860_1036 (size=45) 2024-11-27T13:40:42,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42991 is added to blk_1073741860_1036 (size=45) 2024-11-27T13:40:42,565 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,www,1732714842283.ed774066130adf09502c61d07d72f5ab.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T13:40:42,565 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing ed774066130adf09502c61d07d72f5ab, disabling compactions & flushes 2024-11-27T13:40:42,565 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,www,1732714842283.ed774066130adf09502c61d07d72f5ab. 2024-11-27T13:40:42,565 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1732714842283.ed774066130adf09502c61d07d72f5ab. 2024-11-27T13:40:42,565 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,www,1732714842283.ed774066130adf09502c61d07d72f5ab. after waiting 0 ms 2024-11-27T13:40:42,565 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,www,1732714842283.ed774066130adf09502c61d07d72f5ab. 2024-11-27T13:40:42,566 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,www,1732714842283.ed774066130adf09502c61d07d72f5ab. 2024-11-27T13:40:42,566 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for ed774066130adf09502c61d07d72f5ab: Waiting for close lock at 1732714842565Disabling compacts and flushes for region at 1732714842565Disabling writes for close at 1732714842565Writing region close event to WAL at 1732714842566 (+1 ms)Closed at 1732714842566 2024-11-27T13:40:42,566 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d64103cc558d5243fdfd1e469558a599, NAME => 'test,yyy,1732714842283.d64103cc558d5243fdfd1e469558a599.', STARTKEY => 'yyy', ENDKEY => 'zzz'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237 2024-11-27T13:40:42,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38955 is added to blk_1073741861_1037 (size=45) 2024-11-27T13:40:42,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37007 is added to blk_1073741861_1037 (size=45) 2024-11-27T13:40:42,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42991 is added to blk_1073741861_1037 (size=45) 2024-11-27T13:40:42,572 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,xxx,1732714842283.5c6bad09b1898b3a3810c7bfeb938da4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T13:40:42,572 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 5c6bad09b1898b3a3810c7bfeb938da4, disabling compactions & flushes 2024-11-27T13:40:42,572 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,xxx,1732714842283.5c6bad09b1898b3a3810c7bfeb938da4. 2024-11-27T13:40:42,572 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1732714842283.5c6bad09b1898b3a3810c7bfeb938da4. 2024-11-27T13:40:42,572 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1732714842283.5c6bad09b1898b3a3810c7bfeb938da4. after waiting 0 ms 2024-11-27T13:40:42,572 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1732714842283.5c6bad09b1898b3a3810c7bfeb938da4. 2024-11-27T13:40:42,572 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,xxx,1732714842283.5c6bad09b1898b3a3810c7bfeb938da4. 2024-11-27T13:40:42,572 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 5c6bad09b1898b3a3810c7bfeb938da4: Waiting for close lock at 1732714842572Disabling compacts and flushes for region at 1732714842572Disabling writes for close at 1732714842572Writing region close event to WAL at 1732714842572Closed at 1732714842572 2024-11-27T13:40:42,573 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 6d41565a29ddd5c9153a0f41aba598ca, NAME => 'test,zzz,1732714842283.6d41565a29ddd5c9153a0f41aba598ca.', STARTKEY => 'zzz', ENDKEY => ''}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237 2024-11-27T13:40:42,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42991 is added to blk_1073741862_1038 (size=45) 2024-11-27T13:40:42,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38955 is added to blk_1073741862_1038 (size=45) 2024-11-27T13:40:42,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37007 is added to blk_1073741862_1038 (size=45) 2024-11-27T13:40:42,578 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,yyy,1732714842283.d64103cc558d5243fdfd1e469558a599.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T13:40:42,578 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing d64103cc558d5243fdfd1e469558a599, disabling compactions & flushes 2024-11-27T13:40:42,578 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,yyy,1732714842283.d64103cc558d5243fdfd1e469558a599. 2024-11-27T13:40:42,578 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1732714842283.d64103cc558d5243fdfd1e469558a599. 2024-11-27T13:40:42,578 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1732714842283.d64103cc558d5243fdfd1e469558a599. after waiting 0 ms 2024-11-27T13:40:42,578 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1732714842283.d64103cc558d5243fdfd1e469558a599. 2024-11-27T13:40:42,578 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,yyy,1732714842283.d64103cc558d5243fdfd1e469558a599. 2024-11-27T13:40:42,578 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for d64103cc558d5243fdfd1e469558a599: Waiting for close lock at 1732714842578Disabling compacts and flushes for region at 1732714842578Disabling writes for close at 1732714842578Writing region close event to WAL at 1732714842578Closed at 1732714842578 2024-11-27T13:40:42,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38955 is added to blk_1073741863_1039 (size=42) 2024-11-27T13:40:42,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42991 is added to blk_1073741863_1039 (size=42) 2024-11-27T13:40:42,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37007 is added to blk_1073741863_1039 (size=42) 2024-11-27T13:40:42,584 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,zzz,1732714842283.6d41565a29ddd5c9153a0f41aba598ca.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T13:40:42,584 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 6d41565a29ddd5c9153a0f41aba598ca, disabling compactions & flushes 2024-11-27T13:40:42,584 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,zzz,1732714842283.6d41565a29ddd5c9153a0f41aba598ca. 2024-11-27T13:40:42,584 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1732714842283.6d41565a29ddd5c9153a0f41aba598ca. 2024-11-27T13:40:42,584 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1732714842283.6d41565a29ddd5c9153a0f41aba598ca. after waiting 0 ms 2024-11-27T13:40:42,584 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1732714842283.6d41565a29ddd5c9153a0f41aba598ca. 2024-11-27T13:40:42,584 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,zzz,1732714842283.6d41565a29ddd5c9153a0f41aba598ca. 2024-11-27T13:40:42,584 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 6d41565a29ddd5c9153a0f41aba598ca: Waiting for close lock at 1732714842584Disabling compacts and flushes for region at 1732714842584Disabling writes for close at 1732714842584Writing region close event to WAL at 1732714842584Closed at 1732714842584 2024-11-27T13:40:42,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T13:40:42,806 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,fff,1732714842283.8fc6390c5b3b9356bc4885792f770cf2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T13:40:42,806 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 8fc6390c5b3b9356bc4885792f770cf2, disabling compactions & flushes 2024-11-27T13:40:42,806 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,fff,1732714842283.8fc6390c5b3b9356bc4885792f770cf2. 2024-11-27T13:40:42,806 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1732714842283.8fc6390c5b3b9356bc4885792f770cf2. 2024-11-27T13:40:42,806 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,fff,1732714842283.8fc6390c5b3b9356bc4885792f770cf2. after waiting 0 ms 2024-11-27T13:40:42,806 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,fff,1732714842283.8fc6390c5b3b9356bc4885792f770cf2. 2024-11-27T13:40:42,806 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,fff,1732714842283.8fc6390c5b3b9356bc4885792f770cf2. 2024-11-27T13:40:42,807 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 8fc6390c5b3b9356bc4885792f770cf2: Waiting for close lock at 1732714842806Disabling compacts and flushes for region at 1732714842806Disabling writes for close at 1732714842806Writing region close event to WAL at 1732714842806Closed at 1732714842806 2024-11-27T13:40:42,811 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T13:40:42,818 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,,1732714842283.51060408ec4ea38afe378df89549aeae.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1732714842811"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732714842811"}]},"ts":"1732714842811"} 2024-11-27T13:40:42,818 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ccc,1732714842283.0aef6a7e4e138b1b9d7f4aea6971acc6.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732714842811"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732714842811"}]},"ts":"1732714842811"} 2024-11-27T13:40:42,818 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,bbb,1732714842283.2b6b183b9bc56b2bc4dc6f8968ee01c8.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732714842811"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732714842811"}]},"ts":"1732714842811"} 2024-11-27T13:40:42,819 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,eee,1732714842283.4d1ad27263331aa341edfa82a041de56.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732714842811"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732714842811"}]},"ts":"1732714842811"} 2024-11-27T13:40:42,819 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ddd,1732714842283.49523b5c318014c3e33791ea6f5e308d.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732714842811"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732714842811"}]},"ts":"1732714842811"} 2024-11-27T13:40:42,819 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,hhh,1732714842283.9ee34ecbedcb7dfdabcb99a67ac21045.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732714842811"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732714842811"}]},"ts":"1732714842811"} 2024-11-27T13:40:42,819 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ggg,1732714842283.85dc78011fc464d1446075ad14a217de.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732714842811"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732714842811"}]},"ts":"1732714842811"} 2024-11-27T13:40:42,819 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,iii,1732714842283.dcf91d39846df82f849110c861f2a7d0.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732714842811"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732714842811"}]},"ts":"1732714842811"} 2024-11-27T13:40:42,820 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,jjj,1732714842283.e7a7c9c02e40375cd974ffe51b6da75b.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732714842811"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732714842811"}]},"ts":"1732714842811"} 2024-11-27T13:40:42,820 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,kkk,1732714842283.eeadb5717c90d7f7a33c12d4883bf2b0.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732714842811"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732714842811"}]},"ts":"1732714842811"} 2024-11-27T13:40:42,820 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,lll,1732714842283.a9a679b7eb3754995869ddbc90163b1c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732714842811"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732714842811"}]},"ts":"1732714842811"} 2024-11-27T13:40:42,820 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,mmm,1732714842283.cccf7ce6a567350b0ae162454b0a9379.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732714842811"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732714842811"}]},"ts":"1732714842811"} 2024-11-27T13:40:42,820 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,nnn,1732714842283.d095bb01efaabc5bf5541d373d5419fa.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732714842811"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732714842811"}]},"ts":"1732714842811"} 2024-11-27T13:40:42,821 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ooo,1732714842283.9c46624524e53a899138ac5a715a35d6.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732714842811"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732714842811"}]},"ts":"1732714842811"} 2024-11-27T13:40:42,821 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ppp,1732714842283.de4deb0c16ab89c9c56fc8912227f1c5.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732714842811"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732714842811"}]},"ts":"1732714842811"} 2024-11-27T13:40:42,821 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,qqq,1732714842283.6b448045aa14a354ef4bc74b08978138.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732714842811"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732714842811"}]},"ts":"1732714842811"} 2024-11-27T13:40:42,821 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,rrr,1732714842283.8700ed52628a2e71e47ec460999f2612.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732714842811"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732714842811"}]},"ts":"1732714842811"} 2024-11-27T13:40:42,821 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,sss,1732714842283.f5f0e25bfe9d22339132aab7b2f16f09.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732714842811"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732714842811"}]},"ts":"1732714842811"} 2024-11-27T13:40:42,821 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ttt,1732714842283.922622f55d3d471d091912fe46d7c8d8.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732714842811"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732714842811"}]},"ts":"1732714842811"} 2024-11-27T13:40:42,822 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,uuu,1732714842283.457728e6ba667138202e8a0dfa04b5b7.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732714842811"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732714842811"}]},"ts":"1732714842811"} 2024-11-27T13:40:42,822 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,vvv,1732714842283.c4f747499d4a9438c3cddbfb1ead0d2b.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732714842811"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732714842811"}]},"ts":"1732714842811"} 2024-11-27T13:40:42,822 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,www,1732714842283.ed774066130adf09502c61d07d72f5ab.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732714842811"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732714842811"}]},"ts":"1732714842811"} 2024-11-27T13:40:42,822 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,xxx,1732714842283.5c6bad09b1898b3a3810c7bfeb938da4.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732714842811"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732714842811"}]},"ts":"1732714842811"} 2024-11-27T13:40:42,822 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,yyy,1732714842283.d64103cc558d5243fdfd1e469558a599.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732714842811"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732714842811"}]},"ts":"1732714842811"} 2024-11-27T13:40:42,822 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,zzz,1732714842283.6d41565a29ddd5c9153a0f41aba598ca.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1732714842811"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732714842811"}]},"ts":"1732714842811"} 2024-11-27T13:40:42,822 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,fff,1732714842283.8fc6390c5b3b9356bc4885792f770cf2.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732714842811"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732714842811"}]},"ts":"1732714842811"} 2024-11-27T13:40:42,863 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 26 regions to meta. 2024-11-27T13:40:42,865 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T13:40:42,868 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732714842865"}]},"ts":"1732714842865"} 2024-11-27T13:40:42,872 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLING in hbase:meta 2024-11-27T13:40:42,873 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {29b1b1ba230f=0} racks are {/default-rack=0} 2024-11-27T13:40:42,891 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-27T13:40:42,891 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-27T13:40:42,891 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-27T13:40:42,891 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-27T13:40:42,891 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-27T13:40:42,891 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-27T13:40:42,891 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-27T13:40:42,891 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-27T13:40:42,891 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-27T13:40:42,891 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-27T13:40:42,894 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=51060408ec4ea38afe378df89549aeae, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0aef6a7e4e138b1b9d7f4aea6971acc6, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=49523b5c318014c3e33791ea6f5e308d, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4d1ad27263331aa341edfa82a041de56, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8fc6390c5b3b9356bc4885792f770cf2, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=85dc78011fc464d1446075ad14a217de, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=dcf91d39846df82f849110c861f2a7d0, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e7a7c9c02e40375cd974ffe51b6da75b, ASSIGN}, {pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=eeadb5717c90d7f7a33c12d4883bf2b0, ASSIGN}, {pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a9a679b7eb3754995869ddbc90163b1c, ASSIGN}, {pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=cccf7ce6a567350b0ae162454b0a9379, ASSIGN}, {pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d095bb01efaabc5bf5541d373d5419fa, ASSIGN}, {pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9c46624524e53a899138ac5a715a35d6, ASSIGN}, {pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=de4deb0c16ab89c9c56fc8912227f1c5, ASSIGN}, {pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6b448045aa14a354ef4bc74b08978138, ASSIGN}, {pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8700ed52628a2e71e47ec460999f2612, ASSIGN}, {pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f5f0e25bfe9d22339132aab7b2f16f09, ASSIGN}, {pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=922622f55d3d471d091912fe46d7c8d8, ASSIGN}, {pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=457728e6ba667138202e8a0dfa04b5b7, ASSIGN}, {pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN}, {pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ed774066130adf09502c61d07d72f5ab, ASSIGN}, {pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5c6bad09b1898b3a3810c7bfeb938da4, ASSIGN}, {pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d64103cc558d5243fdfd1e469558a599, ASSIGN}, {pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6d41565a29ddd5c9153a0f41aba598ca, ASSIGN}] 2024-11-27T13:40:42,903 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6d41565a29ddd5c9153a0f41aba598ca, ASSIGN 2024-11-27T13:40:42,904 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=6d41565a29ddd5c9153a0f41aba598ca, ASSIGN; state=OFFLINE, location=29b1b1ba230f,38845,1732714839599; forceNewPlan=false, retain=false 2024-11-27T13:40:42,906 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d64103cc558d5243fdfd1e469558a599, ASSIGN 2024-11-27T13:40:42,908 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d64103cc558d5243fdfd1e469558a599, ASSIGN; state=OFFLINE, location=29b1b1ba230f,39411,1732714839711; forceNewPlan=false, retain=false 2024-11-27T13:40:42,909 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5c6bad09b1898b3a3810c7bfeb938da4, ASSIGN 2024-11-27T13:40:42,911 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=5c6bad09b1898b3a3810c7bfeb938da4, ASSIGN; state=OFFLINE, location=29b1b1ba230f,44709,1732714839765; forceNewPlan=false, retain=false 2024-11-27T13:40:42,912 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ed774066130adf09502c61d07d72f5ab, ASSIGN 2024-11-27T13:40:42,914 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ed774066130adf09502c61d07d72f5ab, ASSIGN; state=OFFLINE, location=29b1b1ba230f,38845,1732714839599; forceNewPlan=false, retain=false 2024-11-27T13:40:42,915 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN 2024-11-27T13:40:42,916 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN; state=OFFLINE, location=29b1b1ba230f,39411,1732714839711; forceNewPlan=false, retain=false 2024-11-27T13:40:42,918 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=457728e6ba667138202e8a0dfa04b5b7, ASSIGN 2024-11-27T13:40:42,919 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=457728e6ba667138202e8a0dfa04b5b7, ASSIGN; state=OFFLINE, location=29b1b1ba230f,44709,1732714839765; forceNewPlan=false, retain=false 2024-11-27T13:40:42,920 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=922622f55d3d471d091912fe46d7c8d8, ASSIGN 2024-11-27T13:40:42,921 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=922622f55d3d471d091912fe46d7c8d8, ASSIGN; state=OFFLINE, location=29b1b1ba230f,38845,1732714839599; forceNewPlan=false, retain=false 2024-11-27T13:40:42,923 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f5f0e25bfe9d22339132aab7b2f16f09, ASSIGN 2024-11-27T13:40:42,924 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f5f0e25bfe9d22339132aab7b2f16f09, ASSIGN; state=OFFLINE, location=29b1b1ba230f,39411,1732714839711; forceNewPlan=false, retain=false 2024-11-27T13:40:42,926 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8700ed52628a2e71e47ec460999f2612, ASSIGN 2024-11-27T13:40:42,927 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8700ed52628a2e71e47ec460999f2612, ASSIGN; state=OFFLINE, location=29b1b1ba230f,44709,1732714839765; forceNewPlan=false, retain=false 2024-11-27T13:40:42,928 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6b448045aa14a354ef4bc74b08978138, ASSIGN 2024-11-27T13:40:42,930 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=6b448045aa14a354ef4bc74b08978138, ASSIGN; state=OFFLINE, location=29b1b1ba230f,38845,1732714839599; forceNewPlan=false, retain=false 2024-11-27T13:40:42,931 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=de4deb0c16ab89c9c56fc8912227f1c5, ASSIGN 2024-11-27T13:40:42,932 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=de4deb0c16ab89c9c56fc8912227f1c5, ASSIGN; state=OFFLINE, location=29b1b1ba230f,39411,1732714839711; forceNewPlan=false, retain=false 2024-11-27T13:40:42,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T13:40:42,933 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9c46624524e53a899138ac5a715a35d6, ASSIGN 2024-11-27T13:40:42,935 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=9c46624524e53a899138ac5a715a35d6, ASSIGN; state=OFFLINE, location=29b1b1ba230f,44709,1732714839765; forceNewPlan=false, retain=false 2024-11-27T13:40:42,936 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d095bb01efaabc5bf5541d373d5419fa, ASSIGN 2024-11-27T13:40:42,937 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d095bb01efaabc5bf5541d373d5419fa, ASSIGN; state=OFFLINE, location=29b1b1ba230f,38845,1732714839599; forceNewPlan=false, retain=false 2024-11-27T13:40:42,938 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=cccf7ce6a567350b0ae162454b0a9379, ASSIGN 2024-11-27T13:40:42,939 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=cccf7ce6a567350b0ae162454b0a9379, ASSIGN; state=OFFLINE, location=29b1b1ba230f,39411,1732714839711; forceNewPlan=false, retain=false 2024-11-27T13:40:42,941 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a9a679b7eb3754995869ddbc90163b1c, ASSIGN 2024-11-27T13:40:42,942 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a9a679b7eb3754995869ddbc90163b1c, ASSIGN; state=OFFLINE, location=29b1b1ba230f,44709,1732714839765; forceNewPlan=false, retain=false 2024-11-27T13:40:42,943 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=eeadb5717c90d7f7a33c12d4883bf2b0, ASSIGN 2024-11-27T13:40:42,944 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=eeadb5717c90d7f7a33c12d4883bf2b0, ASSIGN; state=OFFLINE, location=29b1b1ba230f,38845,1732714839599; forceNewPlan=false, retain=false 2024-11-27T13:40:42,946 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e7a7c9c02e40375cd974ffe51b6da75b, ASSIGN 2024-11-27T13:40:42,947 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=e7a7c9c02e40375cd974ffe51b6da75b, ASSIGN; state=OFFLINE, location=29b1b1ba230f,39411,1732714839711; forceNewPlan=false, retain=false 2024-11-27T13:40:42,948 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=dcf91d39846df82f849110c861f2a7d0, ASSIGN 2024-11-27T13:40:42,949 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=dcf91d39846df82f849110c861f2a7d0, ASSIGN; state=OFFLINE, location=29b1b1ba230f,44709,1732714839765; forceNewPlan=false, retain=false 2024-11-27T13:40:42,950 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN 2024-11-27T13:40:42,951 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN; state=OFFLINE, location=29b1b1ba230f,39411,1732714839711; forceNewPlan=false, retain=false 2024-11-27T13:40:42,952 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=85dc78011fc464d1446075ad14a217de, ASSIGN 2024-11-27T13:40:42,954 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=85dc78011fc464d1446075ad14a217de, ASSIGN; state=OFFLINE, location=29b1b1ba230f,38845,1732714839599; forceNewPlan=false, retain=false 2024-11-27T13:40:42,955 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8fc6390c5b3b9356bc4885792f770cf2, ASSIGN 2024-11-27T13:40:42,956 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8fc6390c5b3b9356bc4885792f770cf2, ASSIGN; state=OFFLINE, location=29b1b1ba230f,44709,1732714839765; forceNewPlan=false, retain=false 2024-11-27T13:40:42,958 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4d1ad27263331aa341edfa82a041de56, ASSIGN 2024-11-27T13:40:42,959 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4d1ad27263331aa341edfa82a041de56, ASSIGN; state=OFFLINE, location=29b1b1ba230f,38845,1732714839599; forceNewPlan=false, retain=false 2024-11-27T13:40:42,960 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=49523b5c318014c3e33791ea6f5e308d, ASSIGN 2024-11-27T13:40:42,961 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=49523b5c318014c3e33791ea6f5e308d, ASSIGN; state=OFFLINE, location=29b1b1ba230f,44709,1732714839765; forceNewPlan=false, retain=false 2024-11-27T13:40:42,962 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0aef6a7e4e138b1b9d7f4aea6971acc6, ASSIGN 2024-11-27T13:40:42,964 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=0aef6a7e4e138b1b9d7f4aea6971acc6, ASSIGN; state=OFFLINE, location=29b1b1ba230f,44709,1732714839765; forceNewPlan=false, retain=false 2024-11-27T13:40:42,965 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN 2024-11-27T13:40:42,966 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN; state=OFFLINE, location=29b1b1ba230f,39411,1732714839711; forceNewPlan=false, retain=false 2024-11-27T13:40:42,968 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=51060408ec4ea38afe378df89549aeae, ASSIGN 2024-11-27T13:40:42,970 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=51060408ec4ea38afe378df89549aeae, ASSIGN; state=OFFLINE, location=29b1b1ba230f,38845,1732714839599; forceNewPlan=false, retain=false 2024-11-27T13:40:43,062 INFO [29b1b1ba230f:35019 {}] balancer.BaseLoadBalancer(388): Reassigned 26 regions. 26 retained the pre-restart assignment. 2024-11-27T13:40:43,064 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=5c6bad09b1898b3a3810c7bfeb938da4, regionState=OPENING, regionLocation=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:43,065 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=457728e6ba667138202e8a0dfa04b5b7, regionState=OPENING, regionLocation=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:43,066 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=8700ed52628a2e71e47ec460999f2612, regionState=OPENING, regionLocation=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:43,067 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=9c46624524e53a899138ac5a715a35d6, regionState=OPENING, regionLocation=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:43,067 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=a9a679b7eb3754995869ddbc90163b1c, regionState=OPENING, regionLocation=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:43,068 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=5c6bad09b1898b3a3810c7bfeb938da4, ASSIGN because future has completed 2024-11-27T13:40:43,068 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=dcf91d39846df82f849110c861f2a7d0, regionState=OPENING, regionLocation=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:43,069 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=8fc6390c5b3b9356bc4885792f770cf2, regionState=OPENING, regionLocation=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:43,070 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5c6bad09b1898b3a3810c7bfeb938da4, server=29b1b1ba230f,44709,1732714839765}] 2024-11-27T13:40:43,071 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=457728e6ba667138202e8a0dfa04b5b7, ASSIGN because future has completed 2024-11-27T13:40:43,073 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8700ed52628a2e71e47ec460999f2612, ASSIGN because future has completed 2024-11-27T13:40:43,075 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=9c46624524e53a899138ac5a715a35d6, ASSIGN because future has completed 2024-11-27T13:40:43,078 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a9a679b7eb3754995869ddbc90163b1c, ASSIGN because future has completed 2024-11-27T13:40:43,079 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure a9a679b7eb3754995869ddbc90163b1c, server=29b1b1ba230f,44709,1732714839765}] 2024-11-27T13:40:43,080 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=dcf91d39846df82f849110c861f2a7d0, ASSIGN because future has completed 2024-11-27T13:40:43,083 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8fc6390c5b3b9356bc4885792f770cf2, ASSIGN because future has completed 2024-11-27T13:40:43,085 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8fc6390c5b3b9356bc4885792f770cf2, server=29b1b1ba230f,44709,1732714839765}] 2024-11-27T13:40:43,091 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure dcf91d39846df82f849110c861f2a7d0, server=29b1b1ba230f,44709,1732714839765}] 2024-11-27T13:40:43,098 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9c46624524e53a899138ac5a715a35d6, server=29b1b1ba230f,44709,1732714839765}] 2024-11-27T13:40:43,104 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8700ed52628a2e71e47ec460999f2612, server=29b1b1ba230f,44709,1732714839765}] 2024-11-27T13:40:43,108 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 457728e6ba667138202e8a0dfa04b5b7, server=29b1b1ba230f,44709,1732714839765}] 2024-11-27T13:40:43,115 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=49523b5c318014c3e33791ea6f5e308d, regionState=OPENING, regionLocation=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:43,116 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=0aef6a7e4e138b1b9d7f4aea6971acc6, regionState=OPENING, regionLocation=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:43,117 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=d64103cc558d5243fdfd1e469558a599, regionState=OPENING, regionLocation=29b1b1ba230f,39411,1732714839711 2024-11-27T13:40:43,118 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=c4f747499d4a9438c3cddbfb1ead0d2b, regionState=OPENING, regionLocation=29b1b1ba230f,39411,1732714839711 2024-11-27T13:40:43,118 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=f5f0e25bfe9d22339132aab7b2f16f09, regionState=OPENING, regionLocation=29b1b1ba230f,39411,1732714839711 2024-11-27T13:40:43,119 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=de4deb0c16ab89c9c56fc8912227f1c5, regionState=OPENING, regionLocation=29b1b1ba230f,39411,1732714839711 2024-11-27T13:40:43,120 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=cccf7ce6a567350b0ae162454b0a9379, regionState=OPENING, regionLocation=29b1b1ba230f,39411,1732714839711 2024-11-27T13:40:43,121 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=e7a7c9c02e40375cd974ffe51b6da75b, regionState=OPENING, regionLocation=29b1b1ba230f,39411,1732714839711 2024-11-27T13:40:43,121 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=9ee34ecbedcb7dfdabcb99a67ac21045, regionState=OPENING, regionLocation=29b1b1ba230f,39411,1732714839711 2024-11-27T13:40:43,122 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=2b6b183b9bc56b2bc4dc6f8968ee01c8, regionState=OPENING, regionLocation=29b1b1ba230f,39411,1732714839711 2024-11-27T13:40:43,123 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=6d41565a29ddd5c9153a0f41aba598ca, regionState=OPENING, regionLocation=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:43,124 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=ed774066130adf09502c61d07d72f5ab, regionState=OPENING, regionLocation=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:43,124 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=49523b5c318014c3e33791ea6f5e308d, ASSIGN because future has completed 2024-11-27T13:40:43,125 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=922622f55d3d471d091912fe46d7c8d8, regionState=OPENING, regionLocation=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:43,126 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 49523b5c318014c3e33791ea6f5e308d, server=29b1b1ba230f,44709,1732714839765}] 2024-11-27T13:40:43,127 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=0aef6a7e4e138b1b9d7f4aea6971acc6, ASSIGN because future has completed 2024-11-27T13:40:43,129 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d64103cc558d5243fdfd1e469558a599, ASSIGN because future has completed 2024-11-27T13:40:43,131 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN because future has completed 2024-11-27T13:40:43,133 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f5f0e25bfe9d22339132aab7b2f16f09, ASSIGN because future has completed 2024-11-27T13:40:43,134 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5f0e25bfe9d22339132aab7b2f16f09, server=29b1b1ba230f,39411,1732714839711}] 2024-11-27T13:40:43,136 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=de4deb0c16ab89c9c56fc8912227f1c5, ASSIGN because future has completed 2024-11-27T13:40:43,138 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=cccf7ce6a567350b0ae162454b0a9379, ASSIGN because future has completed 2024-11-27T13:40:43,140 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=e7a7c9c02e40375cd974ffe51b6da75b, ASSIGN because future has completed 2024-11-27T13:40:43,141 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure e7a7c9c02e40375cd974ffe51b6da75b, server=29b1b1ba230f,39411,1732714839711}] 2024-11-27T13:40:43,142 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN because future has completed 2024-11-27T13:40:43,144 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN because future has completed 2024-11-27T13:40:43,146 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=6d41565a29ddd5c9153a0f41aba598ca, ASSIGN because future has completed 2024-11-27T13:40:43,147 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6d41565a29ddd5c9153a0f41aba598ca, server=29b1b1ba230f,38845,1732714839599}] 2024-11-27T13:40:43,147 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ed774066130adf09502c61d07d72f5ab, ASSIGN because future has completed 2024-11-27T13:40:43,149 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=922622f55d3d471d091912fe46d7c8d8, ASSIGN because future has completed 2024-11-27T13:40:43,154 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 922622f55d3d471d091912fe46d7c8d8, server=29b1b1ba230f,38845,1732714839599}] 2024-11-27T13:40:43,160 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure ed774066130adf09502c61d07d72f5ab, server=29b1b1ba230f,38845,1732714839599}] 2024-11-27T13:40:43,166 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b6b183b9bc56b2bc4dc6f8968ee01c8, server=29b1b1ba230f,39411,1732714839711}] 2024-11-27T13:40:43,171 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9ee34ecbedcb7dfdabcb99a67ac21045, server=29b1b1ba230f,39411,1732714839711}] 2024-11-27T13:40:43,175 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure cccf7ce6a567350b0ae162454b0a9379, server=29b1b1ba230f,39411,1732714839711}] 2024-11-27T13:40:43,179 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,39411,1732714839711}] 2024-11-27T13:40:43,183 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure c4f747499d4a9438c3cddbfb1ead0d2b, server=29b1b1ba230f,39411,1732714839711}] 2024-11-27T13:40:43,187 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure d64103cc558d5243fdfd1e469558a599, server=29b1b1ba230f,39411,1732714839711}] 2024-11-27T13:40:43,191 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0aef6a7e4e138b1b9d7f4aea6971acc6, server=29b1b1ba230f,44709,1732714839765}] 2024-11-27T13:40:43,195 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=6b448045aa14a354ef4bc74b08978138, regionState=OPENING, regionLocation=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:43,196 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=d095bb01efaabc5bf5541d373d5419fa, regionState=OPENING, regionLocation=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:43,197 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=eeadb5717c90d7f7a33c12d4883bf2b0, regionState=OPENING, regionLocation=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:43,197 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=85dc78011fc464d1446075ad14a217de, regionState=OPENING, regionLocation=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:43,198 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=4d1ad27263331aa341edfa82a041de56, regionState=OPENING, regionLocation=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:43,198 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=6b448045aa14a354ef4bc74b08978138, ASSIGN because future has completed 2024-11-27T13:40:43,198 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=51060408ec4ea38afe378df89549aeae, regionState=OPENING, regionLocation=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:43,199 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6b448045aa14a354ef4bc74b08978138, server=29b1b1ba230f,38845,1732714839599}] 2024-11-27T13:40:43,200 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d095bb01efaabc5bf5541d373d5419fa, ASSIGN because future has completed 2024-11-27T13:40:43,201 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=eeadb5717c90d7f7a33c12d4883bf2b0, ASSIGN because future has completed 2024-11-27T13:40:43,202 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=85dc78011fc464d1446075ad14a217de, ASSIGN because future has completed 2024-11-27T13:40:43,203 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4d1ad27263331aa341edfa82a041de56, ASSIGN because future has completed 2024-11-27T13:40:43,204 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4d1ad27263331aa341edfa82a041de56, server=29b1b1ba230f,38845,1732714839599}] 2024-11-27T13:40:43,205 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=51060408ec4ea38afe378df89549aeae, ASSIGN because future has completed 2024-11-27T13:40:43,209 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 51060408ec4ea38afe378df89549aeae, server=29b1b1ba230f,38845,1732714839599}] 2024-11-27T13:40:43,213 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 85dc78011fc464d1446075ad14a217de, server=29b1b1ba230f,38845,1732714839599}] 2024-11-27T13:40:43,217 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure eeadb5717c90d7f7a33c12d4883bf2b0, server=29b1b1ba230f,38845,1732714839599}] 2024-11-27T13:40:43,220 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure d095bb01efaabc5bf5541d373d5419fa, server=29b1b1ba230f,38845,1732714839599}] 2024-11-27T13:40:43,225 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-27T13:40:43,227 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34355, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-27T13:40:43,232 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open test,iii,1732714842283.dcf91d39846df82f849110c861f2a7d0. 2024-11-27T13:40:43,232 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => dcf91d39846df82f849110c861f2a7d0, NAME => 'test,iii,1732714842283.dcf91d39846df82f849110c861f2a7d0.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-27T13:40:43,233 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test dcf91d39846df82f849110c861f2a7d0 2024-11-27T13:40:43,233 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated test,iii,1732714842283.dcf91d39846df82f849110c861f2a7d0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T13:40:43,233 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for dcf91d39846df82f849110c861f2a7d0 2024-11-27T13:40:43,233 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for dcf91d39846df82f849110c861f2a7d0 2024-11-27T13:40:43,235 INFO [StoreOpener-dcf91d39846df82f849110c861f2a7d0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region dcf91d39846df82f849110c861f2a7d0 2024-11-27T13:40:43,237 INFO [StoreOpener-dcf91d39846df82f849110c861f2a7d0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dcf91d39846df82f849110c861f2a7d0 columnFamilyName family 2024-11-27T13:40:43,237 DEBUG [StoreOpener-dcf91d39846df82f849110c861f2a7d0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T13:40:43,238 INFO [StoreOpener-dcf91d39846df82f849110c861f2a7d0-1 {}] regionserver.HStore(327): Store=dcf91d39846df82f849110c861f2a7d0/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T13:40:43,239 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for dcf91d39846df82f849110c861f2a7d0 2024-11-27T13:40:43,240 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/dcf91d39846df82f849110c861f2a7d0 2024-11-27T13:40:43,240 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/dcf91d39846df82f849110c861f2a7d0 2024-11-27T13:40:43,241 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for dcf91d39846df82f849110c861f2a7d0 2024-11-27T13:40:43,241 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for dcf91d39846df82f849110c861f2a7d0 2024-11-27T13:40:43,244 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for dcf91d39846df82f849110c861f2a7d0 2024-11-27T13:40:43,247 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/dcf91d39846df82f849110c861f2a7d0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T13:40:43,248 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened dcf91d39846df82f849110c861f2a7d0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62222626, jitterRate=-0.07281062006950378}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T13:40:43,248 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for dcf91d39846df82f849110c861f2a7d0 2024-11-27T13:40:43,250 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for dcf91d39846df82f849110c861f2a7d0: Running coprocessor pre-open hook at 1732714843233Writing region info on filesystem at 1732714843233Initializing all the Stores at 1732714843235 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732714843235Cleaning up temporary data from old regions at 1732714843241 (+6 ms)Running coprocessor post-open hooks at 1732714843248 (+7 ms)Region opened successfully at 1732714843250 (+2 ms) 2024-11-27T13:40:43,252 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1732714842283.dcf91d39846df82f849110c861f2a7d0., pid=34, masterSystemTime=1732714843225 2024-11-27T13:40:43,255 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for test,iii,1732714842283.dcf91d39846df82f849110c861f2a7d0. 2024-11-27T13:40:43,255 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened test,iii,1732714842283.dcf91d39846df82f849110c861f2a7d0. 2024-11-27T13:40:43,255 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open test,xxx,1732714842283.5c6bad09b1898b3a3810c7bfeb938da4. 2024-11-27T13:40:43,256 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => 5c6bad09b1898b3a3810c7bfeb938da4, NAME => 'test,xxx,1732714842283.5c6bad09b1898b3a3810c7bfeb938da4.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-11-27T13:40:43,256 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 5c6bad09b1898b3a3810c7bfeb938da4 2024-11-27T13:40:43,256 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated test,xxx,1732714842283.5c6bad09b1898b3a3810c7bfeb938da4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T13:40:43,256 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=dcf91d39846df82f849110c861f2a7d0, regionState=OPEN, openSeqNum=2, regionLocation=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:43,256 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for 5c6bad09b1898b3a3810c7bfeb938da4 2024-11-27T13:40:43,256 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for 5c6bad09b1898b3a3810c7bfeb938da4 2024-11-27T13:40:43,259 INFO [StoreOpener-5c6bad09b1898b3a3810c7bfeb938da4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 5c6bad09b1898b3a3810c7bfeb938da4 2024-11-27T13:40:43,260 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure dcf91d39846df82f849110c861f2a7d0, server=29b1b1ba230f,44709,1732714839765 because future has completed 2024-11-27T13:40:43,261 INFO [StoreOpener-5c6bad09b1898b3a3810c7bfeb938da4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5c6bad09b1898b3a3810c7bfeb938da4 columnFamilyName family 2024-11-27T13:40:43,261 DEBUG [StoreOpener-5c6bad09b1898b3a3810c7bfeb938da4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T13:40:43,262 INFO [StoreOpener-5c6bad09b1898b3a3810c7bfeb938da4-1 {}] regionserver.HStore(327): Store=5c6bad09b1898b3a3810c7bfeb938da4/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T13:40:43,263 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for 5c6bad09b1898b3a3810c7bfeb938da4 2024-11-27T13:40:43,264 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/5c6bad09b1898b3a3810c7bfeb938da4 2024-11-27T13:40:43,265 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/5c6bad09b1898b3a3810c7bfeb938da4 2024-11-27T13:40:43,266 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for 5c6bad09b1898b3a3810c7bfeb938da4 2024-11-27T13:40:43,266 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for 5c6bad09b1898b3a3810c7bfeb938da4 2024-11-27T13:40:43,266 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=13 2024-11-27T13:40:43,266 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure dcf91d39846df82f849110c861f2a7d0, server=29b1b1ba230f,44709,1732714839765 in 171 msec 2024-11-27T13:40:43,269 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for 5c6bad09b1898b3a3810c7bfeb938da4 2024-11-27T13:40:43,269 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=dcf91d39846df82f849110c861f2a7d0, ASSIGN in 373 msec 2024-11-27T13:40:43,272 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/5c6bad09b1898b3a3810c7bfeb938da4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T13:40:43,273 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened 5c6bad09b1898b3a3810c7bfeb938da4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64399686, jitterRate=-0.040369898080825806}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T13:40:43,273 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5c6bad09b1898b3a3810c7bfeb938da4 2024-11-27T13:40:43,273 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for 5c6bad09b1898b3a3810c7bfeb938da4: Running coprocessor pre-open hook at 1732714843257Writing region info on filesystem at 1732714843257Initializing all the Stores at 1732714843258 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732714843258Cleaning up temporary data from old regions at 1732714843266 (+8 ms)Running coprocessor post-open hooks at 1732714843273 (+7 ms)Region opened successfully at 1732714843273 2024-11-27T13:40:43,274 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1732714842283.5c6bad09b1898b3a3810c7bfeb938da4., pid=31, masterSystemTime=1732714843225 2024-11-27T13:40:43,277 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1732714842283.5c6bad09b1898b3a3810c7bfeb938da4. 2024-11-27T13:40:43,278 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened test,xxx,1732714842283.5c6bad09b1898b3a3810c7bfeb938da4. 2024-11-27T13:40:43,278 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open test,lll,1732714842283.a9a679b7eb3754995869ddbc90163b1c. 2024-11-27T13:40:43,278 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => a9a679b7eb3754995869ddbc90163b1c, NAME => 'test,lll,1732714842283.a9a679b7eb3754995869ddbc90163b1c.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-27T13:40:43,278 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=5c6bad09b1898b3a3810c7bfeb938da4, regionState=OPEN, openSeqNum=2, regionLocation=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:43,279 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test a9a679b7eb3754995869ddbc90163b1c 2024-11-27T13:40:43,279 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated test,lll,1732714842283.a9a679b7eb3754995869ddbc90163b1c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T13:40:43,279 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for a9a679b7eb3754995869ddbc90163b1c 2024-11-27T13:40:43,279 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for a9a679b7eb3754995869ddbc90163b1c 2024-11-27T13:40:43,281 INFO [StoreOpener-a9a679b7eb3754995869ddbc90163b1c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region a9a679b7eb3754995869ddbc90163b1c 2024-11-27T13:40:43,281 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5c6bad09b1898b3a3810c7bfeb938da4, server=29b1b1ba230f,44709,1732714839765 because future has completed 2024-11-27T13:40:43,284 INFO [StoreOpener-a9a679b7eb3754995869ddbc90163b1c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a9a679b7eb3754995869ddbc90163b1c columnFamilyName family 2024-11-27T13:40:43,284 DEBUG [StoreOpener-a9a679b7eb3754995869ddbc90163b1c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T13:40:43,284 INFO [StoreOpener-a9a679b7eb3754995869ddbc90163b1c-1 {}] regionserver.HStore(327): Store=a9a679b7eb3754995869ddbc90163b1c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T13:40:43,285 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for a9a679b7eb3754995869ddbc90163b1c 2024-11-27T13:40:43,286 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=28 2024-11-27T13:40:43,286 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=28, state=SUCCESS, hasLock=false; OpenRegionProcedure 5c6bad09b1898b3a3810c7bfeb938da4, server=29b1b1ba230f,44709,1732714839765 in 213 msec 2024-11-27T13:40:43,286 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/a9a679b7eb3754995869ddbc90163b1c 2024-11-27T13:40:43,287 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/a9a679b7eb3754995869ddbc90163b1c 2024-11-27T13:40:43,287 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for a9a679b7eb3754995869ddbc90163b1c 2024-11-27T13:40:43,287 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for a9a679b7eb3754995869ddbc90163b1c 2024-11-27T13:40:43,289 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=5c6bad09b1898b3a3810c7bfeb938da4, ASSIGN in 393 msec 2024-11-27T13:40:43,290 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for a9a679b7eb3754995869ddbc90163b1c 2024-11-27T13:40:43,293 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/a9a679b7eb3754995869ddbc90163b1c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T13:40:43,293 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened a9a679b7eb3754995869ddbc90163b1c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69718402, jitterRate=0.038885146379470825}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T13:40:43,294 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a9a679b7eb3754995869ddbc90163b1c 2024-11-27T13:40:43,294 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open test,jjj,1732714842283.e7a7c9c02e40375cd974ffe51b6da75b. 2024-11-27T13:40:43,294 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for a9a679b7eb3754995869ddbc90163b1c: Running coprocessor pre-open hook at 1732714843279Writing region info on filesystem at 1732714843279Initializing all the Stores at 1732714843281 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732714843281Cleaning up temporary data from old regions at 1732714843287 (+6 ms)Running coprocessor post-open hooks at 1732714843294 (+7 ms)Region opened successfully at 1732714843294 2024-11-27T13:40:43,294 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => e7a7c9c02e40375cd974ffe51b6da75b, NAME => 'test,jjj,1732714842283.e7a7c9c02e40375cd974ffe51b6da75b.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-27T13:40:43,294 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test e7a7c9c02e40375cd974ffe51b6da75b 2024-11-27T13:40:43,295 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated test,jjj,1732714842283.e7a7c9c02e40375cd974ffe51b6da75b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T13:40:43,295 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for e7a7c9c02e40375cd974ffe51b6da75b 2024-11-27T13:40:43,295 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for e7a7c9c02e40375cd974ffe51b6da75b 2024-11-27T13:40:43,295 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1732714842283.a9a679b7eb3754995869ddbc90163b1c., pid=32, masterSystemTime=1732714843225 2024-11-27T13:40:43,297 INFO [StoreOpener-e7a7c9c02e40375cd974ffe51b6da75b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region e7a7c9c02e40375cd974ffe51b6da75b 2024-11-27T13:40:43,298 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1732714842283.a9a679b7eb3754995869ddbc90163b1c. 2024-11-27T13:40:43,298 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened test,lll,1732714842283.a9a679b7eb3754995869ddbc90163b1c. 2024-11-27T13:40:43,299 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open test,ddd,1732714842283.49523b5c318014c3e33791ea6f5e308d. 2024-11-27T13:40:43,299 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => 49523b5c318014c3e33791ea6f5e308d, NAME => 'test,ddd,1732714842283.49523b5c318014c3e33791ea6f5e308d.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-11-27T13:40:43,299 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=a9a679b7eb3754995869ddbc90163b1c, regionState=OPEN, openSeqNum=2, regionLocation=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:43,299 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 49523b5c318014c3e33791ea6f5e308d 2024-11-27T13:40:43,299 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated test,ddd,1732714842283.49523b5c318014c3e33791ea6f5e308d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T13:40:43,300 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for 49523b5c318014c3e33791ea6f5e308d 2024-11-27T13:40:43,300 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for 49523b5c318014c3e33791ea6f5e308d 2024-11-27T13:40:43,300 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-27T13:40:43,301 INFO [StoreOpener-e7a7c9c02e40375cd974ffe51b6da75b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7a7c9c02e40375cd974ffe51b6da75b columnFamilyName family 2024-11-27T13:40:43,301 DEBUG [StoreOpener-e7a7c9c02e40375cd974ffe51b6da75b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T13:40:43,302 INFO [StoreOpener-49523b5c318014c3e33791ea6f5e308d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 49523b5c318014c3e33791ea6f5e308d 2024-11-27T13:40:43,303 INFO [StoreOpener-e7a7c9c02e40375cd974ffe51b6da75b-1 {}] regionserver.HStore(327): Store=e7a7c9c02e40375cd974ffe51b6da75b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T13:40:43,303 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure a9a679b7eb3754995869ddbc90163b1c, server=29b1b1ba230f,44709,1732714839765 because future has completed 2024-11-27T13:40:43,303 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40177, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-27T13:40:43,303 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for e7a7c9c02e40375cd974ffe51b6da75b 2024-11-27T13:40:43,304 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/e7a7c9c02e40375cd974ffe51b6da75b 2024-11-27T13:40:43,305 INFO [StoreOpener-49523b5c318014c3e33791ea6f5e308d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 49523b5c318014c3e33791ea6f5e308d columnFamilyName family 2024-11-27T13:40:43,305 DEBUG [StoreOpener-49523b5c318014c3e33791ea6f5e308d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T13:40:43,305 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/e7a7c9c02e40375cd974ffe51b6da75b 2024-11-27T13:40:43,306 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for e7a7c9c02e40375cd974ffe51b6da75b 2024-11-27T13:40:43,306 INFO [StoreOpener-49523b5c318014c3e33791ea6f5e308d-1 {}] regionserver.HStore(327): Store=49523b5c318014c3e33791ea6f5e308d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T13:40:43,306 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for e7a7c9c02e40375cd974ffe51b6da75b 2024-11-27T13:40:43,307 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for 49523b5c318014c3e33791ea6f5e308d 2024-11-27T13:40:43,308 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/49523b5c318014c3e33791ea6f5e308d 2024-11-27T13:40:43,309 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open test,www,1732714842283.ed774066130adf09502c61d07d72f5ab. 2024-11-27T13:40:43,309 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/49523b5c318014c3e33791ea6f5e308d 2024-11-27T13:40:43,309 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => ed774066130adf09502c61d07d72f5ab, NAME => 'test,www,1732714842283.ed774066130adf09502c61d07d72f5ab.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-11-27T13:40:43,309 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ed774066130adf09502c61d07d72f5ab 2024-11-27T13:40:43,310 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated test,www,1732714842283.ed774066130adf09502c61d07d72f5ab.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T13:40:43,310 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for 49523b5c318014c3e33791ea6f5e308d 2024-11-27T13:40:43,310 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for ed774066130adf09502c61d07d72f5ab 2024-11-27T13:40:43,310 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for 49523b5c318014c3e33791ea6f5e308d 2024-11-27T13:40:43,310 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for e7a7c9c02e40375cd974ffe51b6da75b 2024-11-27T13:40:43,310 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for ed774066130adf09502c61d07d72f5ab 2024-11-27T13:40:43,311 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=16 2024-11-27T13:40:43,311 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure a9a679b7eb3754995869ddbc90163b1c, server=29b1b1ba230f,44709,1732714839765 in 226 msec 2024-11-27T13:40:43,312 INFO [StoreOpener-ed774066130adf09502c61d07d72f5ab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ed774066130adf09502c61d07d72f5ab 2024-11-27T13:40:43,313 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for 49523b5c318014c3e33791ea6f5e308d 2024-11-27T13:40:43,315 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/e7a7c9c02e40375cd974ffe51b6da75b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T13:40:43,316 INFO [StoreOpener-ed774066130adf09502c61d07d72f5ab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ed774066130adf09502c61d07d72f5ab columnFamilyName family 2024-11-27T13:40:43,316 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a9a679b7eb3754995869ddbc90163b1c, ASSIGN in 418 msec 2024-11-27T13:40:43,316 DEBUG [StoreOpener-ed774066130adf09502c61d07d72f5ab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T13:40:43,317 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened e7a7c9c02e40375cd974ffe51b6da75b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71213091, jitterRate=0.061157748103141785}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T13:40:43,317 INFO [StoreOpener-ed774066130adf09502c61d07d72f5ab-1 {}] regionserver.HStore(327): Store=ed774066130adf09502c61d07d72f5ab/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T13:40:43,317 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e7a7c9c02e40375cd974ffe51b6da75b 2024-11-27T13:40:43,317 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for ed774066130adf09502c61d07d72f5ab 2024-11-27T13:40:43,318 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for e7a7c9c02e40375cd974ffe51b6da75b: Running coprocessor pre-open hook at 1732714843295Writing region info on filesystem at 1732714843295Initializing all the Stores at 1732714843297 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732714843297Cleaning up temporary data from old regions at 1732714843306 (+9 ms)Running coprocessor post-open hooks at 1732714843317 (+11 ms)Region opened successfully at 1732714843318 (+1 ms) 2024-11-27T13:40:43,318 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/49523b5c318014c3e33791ea6f5e308d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T13:40:43,319 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened 49523b5c318014c3e33791ea6f5e308d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73729153, jitterRate=0.0986499935388565}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T13:40:43,319 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 49523b5c318014c3e33791ea6f5e308d 2024-11-27T13:40:43,319 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/ed774066130adf09502c61d07d72f5ab 2024-11-27T13:40:43,319 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for 49523b5c318014c3e33791ea6f5e308d: Running coprocessor pre-open hook at 1732714843300Writing region info on filesystem at 1732714843300Initializing all the Stores at 1732714843302 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732714843302Cleaning up temporary data from old regions at 1732714843310 (+8 ms)Running coprocessor post-open hooks at 1732714843319 (+9 ms)Region opened successfully at 1732714843319 2024-11-27T13:40:43,320 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for test,jjj,1732714842283.e7a7c9c02e40375cd974ffe51b6da75b., pid=40, masterSystemTime=1732714843288 2024-11-27T13:40:43,320 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/ed774066130adf09502c61d07d72f5ab 2024-11-27T13:40:43,321 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ddd,1732714842283.49523b5c318014c3e33791ea6f5e308d., pid=38, masterSystemTime=1732714843225 2024-11-27T13:40:43,321 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for ed774066130adf09502c61d07d72f5ab 2024-11-27T13:40:43,321 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for ed774066130adf09502c61d07d72f5ab 2024-11-27T13:40:43,323 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for test,jjj,1732714842283.e7a7c9c02e40375cd974ffe51b6da75b. 2024-11-27T13:40:43,324 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened test,jjj,1732714842283.e7a7c9c02e40375cd974ffe51b6da75b. 2024-11-27T13:40:43,324 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(132): Open test,bbb,1732714842283.2b6b183b9bc56b2bc4dc6f8968ee01c8. 2024-11-27T13:40:43,324 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7752): Opening region: {ENCODED => 2b6b183b9bc56b2bc4dc6f8968ee01c8, NAME => 'test,bbb,1732714842283.2b6b183b9bc56b2bc4dc6f8968ee01c8.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-11-27T13:40:43,324 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=e7a7c9c02e40375cd974ffe51b6da75b, regionState=OPEN, openSeqNum=2, regionLocation=29b1b1ba230f,39411,1732714839711 2024-11-27T13:40:43,324 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 2b6b183b9bc56b2bc4dc6f8968ee01c8 2024-11-27T13:40:43,324 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(898): Instantiated test,bbb,1732714842283.2b6b183b9bc56b2bc4dc6f8968ee01c8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T13:40:43,324 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ddd,1732714842283.49523b5c318014c3e33791ea6f5e308d. 2024-11-27T13:40:43,325 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened test,ddd,1732714842283.49523b5c318014c3e33791ea6f5e308d. 2024-11-27T13:40:43,325 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7794): checking encryption for 2b6b183b9bc56b2bc4dc6f8968ee01c8 2024-11-27T13:40:43,325 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7797): checking classloading for 2b6b183b9bc56b2bc4dc6f8968ee01c8 2024-11-27T13:40:43,325 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open test,fff,1732714842283.8fc6390c5b3b9356bc4885792f770cf2. 2024-11-27T13:40:43,325 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 8fc6390c5b3b9356bc4885792f770cf2, NAME => 'test,fff,1732714842283.8fc6390c5b3b9356bc4885792f770cf2.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-27T13:40:43,325 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8fc6390c5b3b9356bc4885792f770cf2 2024-11-27T13:40:43,325 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated test,fff,1732714842283.8fc6390c5b3b9356bc4885792f770cf2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T13:40:43,325 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 8fc6390c5b3b9356bc4885792f770cf2 2024-11-27T13:40:43,326 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 8fc6390c5b3b9356bc4885792f770cf2 2024-11-27T13:40:43,326 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for ed774066130adf09502c61d07d72f5ab 2024-11-27T13:40:43,327 INFO [StoreOpener-2b6b183b9bc56b2bc4dc6f8968ee01c8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 2b6b183b9bc56b2bc4dc6f8968ee01c8 2024-11-27T13:40:43,328 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure e7a7c9c02e40375cd974ffe51b6da75b, server=29b1b1ba230f,39411,1732714839711 because future has completed 2024-11-27T13:40:43,329 INFO [StoreOpener-8fc6390c5b3b9356bc4885792f770cf2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8fc6390c5b3b9356bc4885792f770cf2 2024-11-27T13:40:43,330 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=49523b5c318014c3e33791ea6f5e308d, regionState=OPEN, openSeqNum=2, regionLocation=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:43,330 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35019 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=29b1b1ba230f,39411,1732714839711, table=test, region=e7a7c9c02e40375cd974ffe51b6da75b. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-27T13:40:43,330 INFO [StoreOpener-2b6b183b9bc56b2bc4dc6f8968ee01c8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2b6b183b9bc56b2bc4dc6f8968ee01c8 columnFamilyName family 2024-11-27T13:40:43,330 DEBUG [StoreOpener-2b6b183b9bc56b2bc4dc6f8968ee01c8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T13:40:43,331 INFO [StoreOpener-8fc6390c5b3b9356bc4885792f770cf2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8fc6390c5b3b9356bc4885792f770cf2 columnFamilyName family 2024-11-27T13:40:43,331 INFO [StoreOpener-2b6b183b9bc56b2bc4dc6f8968ee01c8-1 {}] regionserver.HStore(327): Store=2b6b183b9bc56b2bc4dc6f8968ee01c8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T13:40:43,332 DEBUG [StoreOpener-8fc6390c5b3b9356bc4885792f770cf2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T13:40:43,332 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1038): replaying wal for 2b6b183b9bc56b2bc4dc6f8968ee01c8 2024-11-27T13:40:43,332 INFO [StoreOpener-8fc6390c5b3b9356bc4885792f770cf2-1 {}] regionserver.HStore(327): Store=8fc6390c5b3b9356bc4885792f770cf2/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T13:40:43,333 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 49523b5c318014c3e33791ea6f5e308d, server=29b1b1ba230f,44709,1732714839765 because future has completed 2024-11-27T13:40:43,333 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 8fc6390c5b3b9356bc4885792f770cf2 2024-11-27T13:40:43,333 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/2b6b183b9bc56b2bc4dc6f8968ee01c8 2024-11-27T13:40:43,333 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/2b6b183b9bc56b2bc4dc6f8968ee01c8 2024-11-27T13:40:43,334 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/8fc6390c5b3b9356bc4885792f770cf2 2024-11-27T13:40:43,334 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1048): stopping wal replay for 2b6b183b9bc56b2bc4dc6f8968ee01c8 2024-11-27T13:40:43,334 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/8fc6390c5b3b9356bc4885792f770cf2 2024-11-27T13:40:43,334 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1060): Cleaning up temporary data for 2b6b183b9bc56b2bc4dc6f8968ee01c8 2024-11-27T13:40:43,335 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 8fc6390c5b3b9356bc4885792f770cf2 2024-11-27T13:40:43,335 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 8fc6390c5b3b9356bc4885792f770cf2 2024-11-27T13:40:43,336 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=14 2024-11-27T13:40:43,336 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure e7a7c9c02e40375cd974ffe51b6da75b, server=29b1b1ba230f,39411,1732714839711 in 192 msec 2024-11-27T13:40:43,337 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1093): writing seq id for 2b6b183b9bc56b2bc4dc6f8968ee01c8 2024-11-27T13:40:43,337 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 8fc6390c5b3b9356bc4885792f770cf2 2024-11-27T13:40:43,339 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e7a7c9c02e40375cd974ffe51b6da75b, ASSIGN in 444 msec 2024-11-27T13:40:43,341 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/ed774066130adf09502c61d07d72f5ab/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T13:40:43,342 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/2b6b183b9bc56b2bc4dc6f8968ee01c8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T13:40:43,342 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened ed774066130adf09502c61d07d72f5ab; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72808258, jitterRate=0.08492758870124817}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T13:40:43,342 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ed774066130adf09502c61d07d72f5ab 2024-11-27T13:40:43,342 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/8fc6390c5b3b9356bc4885792f770cf2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T13:40:43,343 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for ed774066130adf09502c61d07d72f5ab: Running coprocessor pre-open hook at 1732714843310Writing region info on filesystem at 1732714843310Initializing all the Stores at 1732714843311 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732714843312 (+1 ms)Cleaning up temporary data from old regions at 1732714843321 (+9 ms)Running coprocessor post-open hooks at 1732714843343 (+22 ms)Region opened successfully at 1732714843343 2024-11-27T13:40:43,344 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1114): Opened 2b6b183b9bc56b2bc4dc6f8968ee01c8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61890942, jitterRate=-0.07775309681892395}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T13:40:43,344 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2b6b183b9bc56b2bc4dc6f8968ee01c8 2024-11-27T13:40:43,344 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1006): Region open journal for 2b6b183b9bc56b2bc4dc6f8968ee01c8: Running coprocessor pre-open hook at 1732714843325Writing region info on filesystem at 1732714843325Initializing all the Stores at 1732714843327 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732714843327Cleaning up temporary data from old regions at 1732714843334 (+7 ms)Running coprocessor post-open hooks at 1732714843344 (+10 ms)Region opened successfully at 1732714843344 2024-11-27T13:40:43,344 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=8 2024-11-27T13:40:43,345 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 49523b5c318014c3e33791ea6f5e308d, server=29b1b1ba230f,44709,1732714839765 in 214 msec 2024-11-27T13:40:43,344 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 8fc6390c5b3b9356bc4885792f770cf2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67226636, jitterRate=0.0017549395561218262}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T13:40:43,345 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8fc6390c5b3b9356bc4885792f770cf2 2024-11-27T13:40:43,345 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 8fc6390c5b3b9356bc4885792f770cf2: Running coprocessor pre-open hook at 1732714843326Writing region info on filesystem at 1732714843326Initializing all the Stores at 1732714843328 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732714843328Cleaning up temporary data from old regions at 1732714843335 (+7 ms)Running coprocessor post-open hooks at 1732714843345 (+10 ms)Region opened successfully at 1732714843345 2024-11-27T13:40:43,346 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for test,www,1732714842283.ed774066130adf09502c61d07d72f5ab., pid=43, masterSystemTime=1732714843300 2024-11-27T13:40:43,346 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2236): Post open deploy tasks for test,bbb,1732714842283.2b6b183b9bc56b2bc4dc6f8968ee01c8., pid=44, masterSystemTime=1732714843288 2024-11-27T13:40:43,347 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1732714842283.8fc6390c5b3b9356bc4885792f770cf2., pid=33, masterSystemTime=1732714843225 2024-11-27T13:40:43,348 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=49523b5c318014c3e33791ea6f5e308d, ASSIGN in 452 msec 2024-11-27T13:40:43,349 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2266): Finished post open deploy task for test,bbb,1732714842283.2b6b183b9bc56b2bc4dc6f8968ee01c8. 2024-11-27T13:40:43,349 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(153): Opened test,bbb,1732714842283.2b6b183b9bc56b2bc4dc6f8968ee01c8. 2024-11-27T13:40:43,349 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open test,sss,1732714842283.f5f0e25bfe9d22339132aab7b2f16f09. 2024-11-27T13:40:43,349 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=2b6b183b9bc56b2bc4dc6f8968ee01c8, regionState=OPEN, openSeqNum=2, regionLocation=29b1b1ba230f,39411,1732714839711 2024-11-27T13:40:43,349 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => f5f0e25bfe9d22339132aab7b2f16f09, NAME => 'test,sss,1732714842283.f5f0e25bfe9d22339132aab7b2f16f09.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-27T13:40:43,350 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f5f0e25bfe9d22339132aab7b2f16f09 2024-11-27T13:40:43,350 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for test,www,1732714842283.ed774066130adf09502c61d07d72f5ab. 2024-11-27T13:40:43,350 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened test,www,1732714842283.ed774066130adf09502c61d07d72f5ab. 2024-11-27T13:40:43,350 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated test,sss,1732714842283.f5f0e25bfe9d22339132aab7b2f16f09.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T13:40:43,350 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for f5f0e25bfe9d22339132aab7b2f16f09 2024-11-27T13:40:43,350 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open test,kkk,1732714842283.eeadb5717c90d7f7a33c12d4883bf2b0. 2024-11-27T13:40:43,350 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for f5f0e25bfe9d22339132aab7b2f16f09 2024-11-27T13:40:43,350 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => eeadb5717c90d7f7a33c12d4883bf2b0, NAME => 'test,kkk,1732714842283.eeadb5717c90d7f7a33c12d4883bf2b0.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-11-27T13:40:43,351 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test eeadb5717c90d7f7a33c12d4883bf2b0 2024-11-27T13:40:43,351 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated test,kkk,1732714842283.eeadb5717c90d7f7a33c12d4883bf2b0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T13:40:43,351 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for eeadb5717c90d7f7a33c12d4883bf2b0 2024-11-27T13:40:43,351 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for eeadb5717c90d7f7a33c12d4883bf2b0 2024-11-27T13:40:43,353 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b6b183b9bc56b2bc4dc6f8968ee01c8, server=29b1b1ba230f,39411,1732714839711 because future has completed 2024-11-27T13:40:43,353 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1732714842283.8fc6390c5b3b9356bc4885792f770cf2. 2024-11-27T13:40:43,353 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened test,fff,1732714842283.8fc6390c5b3b9356bc4885792f770cf2. 2024-11-27T13:40:43,354 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open test,ooo,1732714842283.9c46624524e53a899138ac5a715a35d6. 2024-11-27T13:40:43,354 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => 9c46624524e53a899138ac5a715a35d6, NAME => 'test,ooo,1732714842283.9c46624524e53a899138ac5a715a35d6.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-11-27T13:40:43,354 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 9c46624524e53a899138ac5a715a35d6 2024-11-27T13:40:43,354 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated test,ooo,1732714842283.9c46624524e53a899138ac5a715a35d6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T13:40:43,354 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for 9c46624524e53a899138ac5a715a35d6 2024-11-27T13:40:43,355 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for 9c46624524e53a899138ac5a715a35d6 2024-11-27T13:40:43,355 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=8fc6390c5b3b9356bc4885792f770cf2, regionState=OPEN, openSeqNum=2, regionLocation=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:43,355 INFO [StoreOpener-eeadb5717c90d7f7a33c12d4883bf2b0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region eeadb5717c90d7f7a33c12d4883bf2b0 2024-11-27T13:40:43,355 INFO [StoreOpener-f5f0e25bfe9d22339132aab7b2f16f09-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f5f0e25bfe9d22339132aab7b2f16f09 2024-11-27T13:40:43,358 INFO [StoreOpener-eeadb5717c90d7f7a33c12d4883bf2b0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eeadb5717c90d7f7a33c12d4883bf2b0 columnFamilyName family 2024-11-27T13:40:43,358 DEBUG [StoreOpener-eeadb5717c90d7f7a33c12d4883bf2b0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T13:40:43,359 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8fc6390c5b3b9356bc4885792f770cf2, server=29b1b1ba230f,44709,1732714839765 because future has completed 2024-11-27T13:40:43,359 INFO [StoreOpener-f5f0e25bfe9d22339132aab7b2f16f09-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f5f0e25bfe9d22339132aab7b2f16f09 columnFamilyName family 2024-11-27T13:40:43,360 DEBUG [StoreOpener-f5f0e25bfe9d22339132aab7b2f16f09-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T13:40:43,360 INFO [StoreOpener-9c46624524e53a899138ac5a715a35d6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 9c46624524e53a899138ac5a715a35d6 2024-11-27T13:40:43,360 INFO [StoreOpener-eeadb5717c90d7f7a33c12d4883bf2b0-1 {}] regionserver.HStore(327): Store=eeadb5717c90d7f7a33c12d4883bf2b0/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T13:40:43,360 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for eeadb5717c90d7f7a33c12d4883bf2b0 2024-11-27T13:40:43,361 INFO [StoreOpener-f5f0e25bfe9d22339132aab7b2f16f09-1 {}] regionserver.HStore(327): Store=f5f0e25bfe9d22339132aab7b2f16f09/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T13:40:43,361 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/eeadb5717c90d7f7a33c12d4883bf2b0 2024-11-27T13:40:43,362 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/eeadb5717c90d7f7a33c12d4883bf2b0 2024-11-27T13:40:43,362 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for f5f0e25bfe9d22339132aab7b2f16f09 2024-11-27T13:40:43,363 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for eeadb5717c90d7f7a33c12d4883bf2b0 2024-11-27T13:40:43,363 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for eeadb5717c90d7f7a33c12d4883bf2b0 2024-11-27T13:40:43,363 INFO [StoreOpener-9c46624524e53a899138ac5a715a35d6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9c46624524e53a899138ac5a715a35d6 columnFamilyName family 2024-11-27T13:40:43,363 DEBUG [StoreOpener-9c46624524e53a899138ac5a715a35d6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T13:40:43,364 INFO [StoreOpener-9c46624524e53a899138ac5a715a35d6-1 {}] regionserver.HStore(327): Store=9c46624524e53a899138ac5a715a35d6/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T13:40:43,364 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for 9c46624524e53a899138ac5a715a35d6 2024-11-27T13:40:43,365 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/f5f0e25bfe9d22339132aab7b2f16f09 2024-11-27T13:40:43,365 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/9c46624524e53a899138ac5a715a35d6 2024-11-27T13:40:43,365 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=6 2024-11-27T13:40:43,365 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure 2b6b183b9bc56b2bc4dc6f8968ee01c8, server=29b1b1ba230f,39411,1732714839711 in 193 msec 2024-11-27T13:40:43,366 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/9c46624524e53a899138ac5a715a35d6 2024-11-27T13:40:43,366 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for eeadb5717c90d7f7a33c12d4883bf2b0 2024-11-27T13:40:43,366 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/f5f0e25bfe9d22339132aab7b2f16f09 2024-11-27T13:40:43,366 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for 9c46624524e53a899138ac5a715a35d6 2024-11-27T13:40:43,367 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for 9c46624524e53a899138ac5a715a35d6 2024-11-27T13:40:43,367 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for f5f0e25bfe9d22339132aab7b2f16f09 2024-11-27T13:40:43,367 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for f5f0e25bfe9d22339132aab7b2f16f09 2024-11-27T13:40:43,369 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN in 473 msec 2024-11-27T13:40:43,370 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/eeadb5717c90d7f7a33c12d4883bf2b0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T13:40:43,370 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for 9c46624524e53a899138ac5a715a35d6 2024-11-27T13:40:43,371 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened eeadb5717c90d7f7a33c12d4883bf2b0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72493603, jitterRate=0.08023886382579803}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T13:40:43,371 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for eeadb5717c90d7f7a33c12d4883bf2b0 2024-11-27T13:40:43,371 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for eeadb5717c90d7f7a33c12d4883bf2b0: Running coprocessor pre-open hook at 1732714843351Writing region info on filesystem at 1732714843351Initializing all the Stores at 1732714843353 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732714843353Cleaning up temporary data from old regions at 1732714843363 (+10 ms)Running coprocessor post-open hooks at 1732714843371 (+8 ms)Region opened successfully at 1732714843371 2024-11-27T13:40:43,372 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for f5f0e25bfe9d22339132aab7b2f16f09 2024-11-27T13:40:43,372 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for test,kkk,1732714842283.eeadb5717c90d7f7a33c12d4883bf2b0., pid=55, masterSystemTime=1732714843300 2024-11-27T13:40:43,375 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for test,kkk,1732714842283.eeadb5717c90d7f7a33c12d4883bf2b0. 2024-11-27T13:40:43,376 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened test,kkk,1732714842283.eeadb5717c90d7f7a33c12d4883bf2b0. 2024-11-27T13:40:43,376 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(132): Open test,eee,1732714842283.4d1ad27263331aa341edfa82a041de56. 2024-11-27T13:40:43,376 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7752): Opening region: {ENCODED => 4d1ad27263331aa341edfa82a041de56, NAME => 'test,eee,1732714842283.4d1ad27263331aa341edfa82a041de56.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-27T13:40:43,376 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4d1ad27263331aa341edfa82a041de56 2024-11-27T13:40:43,376 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(898): Instantiated test,eee,1732714842283.4d1ad27263331aa341edfa82a041de56.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T13:40:43,377 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7794): checking encryption for 4d1ad27263331aa341edfa82a041de56 2024-11-27T13:40:43,377 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7797): checking classloading for 4d1ad27263331aa341edfa82a041de56 2024-11-27T13:40:43,378 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=10 2024-11-27T13:40:43,378 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 8fc6390c5b3b9356bc4885792f770cf2, server=29b1b1ba230f,44709,1732714839765 in 287 msec 2024-11-27T13:40:43,381 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8fc6390c5b3b9356bc4885792f770cf2, ASSIGN in 485 msec 2024-11-27T13:40:43,381 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/9c46624524e53a899138ac5a715a35d6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T13:40:43,382 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened 9c46624524e53a899138ac5a715a35d6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60529881, jitterRate=-0.09803448617458344}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T13:40:43,383 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9c46624524e53a899138ac5a715a35d6 2024-11-27T13:40:43,383 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=eeadb5717c90d7f7a33c12d4883bf2b0, regionState=OPEN, openSeqNum=2, regionLocation=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:43,383 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for 9c46624524e53a899138ac5a715a35d6: Running coprocessor pre-open hook at 1732714843355Writing region info on filesystem at 1732714843355Initializing all the Stores at 1732714843357 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732714843357Cleaning up temporary data from old regions at 1732714843367 (+10 ms)Running coprocessor post-open hooks at 1732714843383 (+16 ms)Region opened successfully at 1732714843383 2024-11-27T13:40:43,384 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ooo,1732714842283.9c46624524e53a899138ac5a715a35d6., pid=35, masterSystemTime=1732714843225 2024-11-27T13:40:43,387 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure eeadb5717c90d7f7a33c12d4883bf2b0, server=29b1b1ba230f,38845,1732714839599 because future has completed 2024-11-27T13:40:43,388 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ooo,1732714842283.9c46624524e53a899138ac5a715a35d6. 2024-11-27T13:40:43,388 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened test,ooo,1732714842283.9c46624524e53a899138ac5a715a35d6. 2024-11-27T13:40:43,388 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(132): Open test,ccc,1732714842283.0aef6a7e4e138b1b9d7f4aea6971acc6. 2024-11-27T13:40:43,389 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7752): Opening region: {ENCODED => 0aef6a7e4e138b1b9d7f4aea6971acc6, NAME => 'test,ccc,1732714842283.0aef6a7e4e138b1b9d7f4aea6971acc6.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-27T13:40:43,389 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 0aef6a7e4e138b1b9d7f4aea6971acc6 2024-11-27T13:40:43,389 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(898): Instantiated test,ccc,1732714842283.0aef6a7e4e138b1b9d7f4aea6971acc6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T13:40:43,389 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7794): checking encryption for 0aef6a7e4e138b1b9d7f4aea6971acc6 2024-11-27T13:40:43,389 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7797): checking classloading for 0aef6a7e4e138b1b9d7f4aea6971acc6 2024-11-27T13:40:43,392 INFO [StoreOpener-4d1ad27263331aa341edfa82a041de56-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4d1ad27263331aa341edfa82a041de56 2024-11-27T13:40:43,393 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/f5f0e25bfe9d22339132aab7b2f16f09/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T13:40:43,393 INFO [StoreOpener-0aef6a7e4e138b1b9d7f4aea6971acc6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 0aef6a7e4e138b1b9d7f4aea6971acc6 2024-11-27T13:40:43,394 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened f5f0e25bfe9d22339132aab7b2f16f09; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69953045, jitterRate=0.04238159954547882}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T13:40:43,394 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f5f0e25bfe9d22339132aab7b2f16f09 2024-11-27T13:40:43,395 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for f5f0e25bfe9d22339132aab7b2f16f09: Running coprocessor pre-open hook at 1732714843350Writing region info on filesystem at 1732714843350Initializing all the Stores at 1732714843353 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732714843353Cleaning up temporary data from old regions at 1732714843367 (+14 ms)Running coprocessor post-open hooks at 1732714843395 (+28 ms)Region opened successfully at 1732714843395 2024-11-27T13:40:43,395 INFO [StoreOpener-4d1ad27263331aa341edfa82a041de56-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4d1ad27263331aa341edfa82a041de56 columnFamilyName family 2024-11-27T13:40:43,395 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=15 2024-11-27T13:40:43,395 DEBUG [StoreOpener-4d1ad27263331aa341edfa82a041de56-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T13:40:43,395 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure eeadb5717c90d7f7a33c12d4883bf2b0, server=29b1b1ba230f,38845,1732714839599 in 174 msec 2024-11-27T13:40:43,396 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1732714842283.f5f0e25bfe9d22339132aab7b2f16f09., pid=39, masterSystemTime=1732714843288 2024-11-27T13:40:43,396 INFO [StoreOpener-4d1ad27263331aa341edfa82a041de56-1 {}] regionserver.HStore(327): Store=4d1ad27263331aa341edfa82a041de56/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T13:40:43,397 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1038): replaying wal for 4d1ad27263331aa341edfa82a041de56 2024-11-27T13:40:43,398 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/4d1ad27263331aa341edfa82a041de56 2024-11-27T13:40:43,399 INFO [StoreOpener-0aef6a7e4e138b1b9d7f4aea6971acc6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0aef6a7e4e138b1b9d7f4aea6971acc6 columnFamilyName family 2024-11-27T13:40:43,399 DEBUG [StoreOpener-0aef6a7e4e138b1b9d7f4aea6971acc6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T13:40:43,399 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/4d1ad27263331aa341edfa82a041de56 2024-11-27T13:40:43,400 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1732714842283.f5f0e25bfe9d22339132aab7b2f16f09. 2024-11-27T13:40:43,400 INFO [StoreOpener-0aef6a7e4e138b1b9d7f4aea6971acc6-1 {}] regionserver.HStore(327): Store=0aef6a7e4e138b1b9d7f4aea6971acc6/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T13:40:43,400 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened test,sss,1732714842283.f5f0e25bfe9d22339132aab7b2f16f09. 2024-11-27T13:40:43,400 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1038): replaying wal for 0aef6a7e4e138b1b9d7f4aea6971acc6 2024-11-27T13:40:43,400 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open test,hhh,1732714842283.9ee34ecbedcb7dfdabcb99a67ac21045. 2024-11-27T13:40:43,401 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1048): stopping wal replay for 4d1ad27263331aa341edfa82a041de56 2024-11-27T13:40:43,401 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1060): Cleaning up temporary data for 4d1ad27263331aa341edfa82a041de56 2024-11-27T13:40:43,400 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => 9ee34ecbedcb7dfdabcb99a67ac21045, NAME => 'test,hhh,1732714842283.9ee34ecbedcb7dfdabcb99a67ac21045.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-27T13:40:43,401 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 9ee34ecbedcb7dfdabcb99a67ac21045 2024-11-27T13:40:43,401 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/0aef6a7e4e138b1b9d7f4aea6971acc6 2024-11-27T13:40:43,402 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated test,hhh,1732714842283.9ee34ecbedcb7dfdabcb99a67ac21045.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T13:40:43,402 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for 9ee34ecbedcb7dfdabcb99a67ac21045 2024-11-27T13:40:43,403 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for 9ee34ecbedcb7dfdabcb99a67ac21045 2024-11-27T13:40:43,403 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/0aef6a7e4e138b1b9d7f4aea6971acc6 2024-11-27T13:40:43,403 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1048): stopping wal replay for 0aef6a7e4e138b1b9d7f4aea6971acc6 2024-11-27T13:40:43,403 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1060): Cleaning up temporary data for 0aef6a7e4e138b1b9d7f4aea6971acc6 2024-11-27T13:40:43,404 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): writing seq id for 4d1ad27263331aa341edfa82a041de56 2024-11-27T13:40:43,406 INFO [StoreOpener-9ee34ecbedcb7dfdabcb99a67ac21045-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 9ee34ecbedcb7dfdabcb99a67ac21045 2024-11-27T13:40:43,407 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1093): writing seq id for 0aef6a7e4e138b1b9d7f4aea6971acc6 2024-11-27T13:40:43,407 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/4d1ad27263331aa341edfa82a041de56/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T13:40:43,408 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1114): Opened 4d1ad27263331aa341edfa82a041de56; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68127558, jitterRate=0.015179723501205444}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T13:40:43,408 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=eeadb5717c90d7f7a33c12d4883bf2b0, ASSIGN in 502 msec 2024-11-27T13:40:43,408 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4d1ad27263331aa341edfa82a041de56 2024-11-27T13:40:43,408 INFO [StoreOpener-9ee34ecbedcb7dfdabcb99a67ac21045-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9ee34ecbedcb7dfdabcb99a67ac21045 columnFamilyName family 2024-11-27T13:40:43,408 DEBUG [StoreOpener-9ee34ecbedcb7dfdabcb99a67ac21045-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T13:40:43,409 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1006): Region open journal for 4d1ad27263331aa341edfa82a041de56: Running coprocessor pre-open hook at 1732714843377Writing region info on filesystem at 1732714843377Initializing all the Stores at 1732714843378 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732714843378Cleaning up temporary data from old regions at 1732714843401 (+23 ms)Running coprocessor post-open hooks at 1732714843408 (+7 ms)Region opened successfully at 1732714843408 2024-11-27T13:40:43,410 INFO [StoreOpener-9ee34ecbedcb7dfdabcb99a67ac21045-1 {}] regionserver.HStore(327): Store=9ee34ecbedcb7dfdabcb99a67ac21045/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T13:40:43,410 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1732714842283.4d1ad27263331aa341edfa82a041de56., pid=52, masterSystemTime=1732714843300 2024-11-27T13:40:43,410 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for 9ee34ecbedcb7dfdabcb99a67ac21045 2024-11-27T13:40:43,411 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=f5f0e25bfe9d22339132aab7b2f16f09, regionState=OPEN, openSeqNum=2, regionLocation=29b1b1ba230f,39411,1732714839711 2024-11-27T13:40:43,413 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/9ee34ecbedcb7dfdabcb99a67ac21045 2024-11-27T13:40:43,414 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/9ee34ecbedcb7dfdabcb99a67ac21045 2024-11-27T13:40:43,414 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5f0e25bfe9d22339132aab7b2f16f09, server=29b1b1ba230f,39411,1732714839711 because future has completed 2024-11-27T13:40:43,414 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/0aef6a7e4e138b1b9d7f4aea6971acc6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T13:40:43,415 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for 9ee34ecbedcb7dfdabcb99a67ac21045 2024-11-27T13:40:43,415 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for 9ee34ecbedcb7dfdabcb99a67ac21045 2024-11-27T13:40:43,415 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1732714842283.4d1ad27263331aa341edfa82a041de56. 2024-11-27T13:40:43,415 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(153): Opened test,eee,1732714842283.4d1ad27263331aa341edfa82a041de56. 2024-11-27T13:40:43,415 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(132): Open test,qqq,1732714842283.6b448045aa14a354ef4bc74b08978138. 2024-11-27T13:40:43,415 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7752): Opening region: {ENCODED => 6b448045aa14a354ef4bc74b08978138, NAME => 'test,qqq,1732714842283.6b448045aa14a354ef4bc74b08978138.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-11-27T13:40:43,416 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1114): Opened 0aef6a7e4e138b1b9d7f4aea6971acc6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75484872, jitterRate=0.12481224536895752}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T13:40:43,416 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 6b448045aa14a354ef4bc74b08978138 2024-11-27T13:40:43,416 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0aef6a7e4e138b1b9d7f4aea6971acc6 2024-11-27T13:40:43,416 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(898): Instantiated test,qqq,1732714842283.6b448045aa14a354ef4bc74b08978138.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T13:40:43,416 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1006): Region open journal for 0aef6a7e4e138b1b9d7f4aea6971acc6: Running coprocessor pre-open hook at 1732714843390Writing region info on filesystem at 1732714843390Initializing all the Stores at 1732714843391 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732714843391Cleaning up temporary data from old regions at 1732714843404 (+13 ms)Running coprocessor post-open hooks at 1732714843416 (+12 ms)Region opened successfully at 1732714843416 2024-11-27T13:40:43,416 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7794): checking encryption for 6b448045aa14a354ef4bc74b08978138 2024-11-27T13:40:43,416 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7797): checking classloading for 6b448045aa14a354ef4bc74b08978138 2024-11-27T13:40:43,417 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=4d1ad27263331aa341edfa82a041de56, regionState=OPEN, openSeqNum=2, regionLocation=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:43,417 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1732714842283.0aef6a7e4e138b1b9d7f4aea6971acc6., pid=50, masterSystemTime=1732714843225 2024-11-27T13:40:43,419 INFO [StoreOpener-6b448045aa14a354ef4bc74b08978138-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 6b448045aa14a354ef4bc74b08978138 2024-11-27T13:40:43,419 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for 9ee34ecbedcb7dfdabcb99a67ac21045 2024-11-27T13:40:43,423 INFO [StoreOpener-6b448045aa14a354ef4bc74b08978138-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6b448045aa14a354ef4bc74b08978138 columnFamilyName family 2024-11-27T13:40:43,423 DEBUG [StoreOpener-6b448045aa14a354ef4bc74b08978138-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T13:40:43,423 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4d1ad27263331aa341edfa82a041de56, server=29b1b1ba230f,38845,1732714839599 because future has completed 2024-11-27T13:40:43,425 INFO [StoreOpener-6b448045aa14a354ef4bc74b08978138-1 {}] regionserver.HStore(327): Store=6b448045aa14a354ef4bc74b08978138/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T13:40:43,425 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1038): replaying wal for 6b448045aa14a354ef4bc74b08978138 2024-11-27T13:40:43,426 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/6b448045aa14a354ef4bc74b08978138 2024-11-27T13:40:43,426 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35019 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=29b1b1ba230f,38845,1732714839599, table=test, region=4d1ad27263331aa341edfa82a041de56. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-27T13:40:43,427 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ccc,1732714842283.0aef6a7e4e138b1b9d7f4aea6971acc6. 2024-11-27T13:40:43,427 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(153): Opened test,ccc,1732714842283.0aef6a7e4e138b1b9d7f4aea6971acc6. 2024-11-27T13:40:43,427 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/6b448045aa14a354ef4bc74b08978138 2024-11-27T13:40:43,427 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open test,rrr,1732714842283.8700ed52628a2e71e47ec460999f2612. 2024-11-27T13:40:43,427 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/9ee34ecbedcb7dfdabcb99a67ac21045/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T13:40:43,428 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => 8700ed52628a2e71e47ec460999f2612, NAME => 'test,rrr,1732714842283.8700ed52628a2e71e47ec460999f2612.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-27T13:40:43,428 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1048): stopping wal replay for 6b448045aa14a354ef4bc74b08978138 2024-11-27T13:40:43,428 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8700ed52628a2e71e47ec460999f2612 2024-11-27T13:40:43,428 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened 9ee34ecbedcb7dfdabcb99a67ac21045; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75419085, jitterRate=0.12383194267749786}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T13:40:43,428 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1060): Cleaning up temporary data for 6b448045aa14a354ef4bc74b08978138 2024-11-27T13:40:43,428 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9ee34ecbedcb7dfdabcb99a67ac21045 2024-11-27T13:40:43,428 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated test,rrr,1732714842283.8700ed52628a2e71e47ec460999f2612.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T13:40:43,429 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for 9ee34ecbedcb7dfdabcb99a67ac21045: Running coprocessor pre-open hook at 1732714843403Writing region info on filesystem at 1732714843403Initializing all the Stores at 1732714843404 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732714843404Cleaning up temporary data from old regions at 1732714843415 (+11 ms)Running coprocessor post-open hooks at 1732714843428 (+13 ms)Region opened successfully at 1732714843428 2024-11-27T13:40:43,429 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for 8700ed52628a2e71e47ec460999f2612 2024-11-27T13:40:43,429 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for 8700ed52628a2e71e47ec460999f2612 2024-11-27T13:40:43,430 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1732714842283.9ee34ecbedcb7dfdabcb99a67ac21045., pid=45, masterSystemTime=1732714843288 2024-11-27T13:40:43,432 INFO [StoreOpener-8700ed52628a2e71e47ec460999f2612-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8700ed52628a2e71e47ec460999f2612 2024-11-27T13:40:43,432 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): writing seq id for 6b448045aa14a354ef4bc74b08978138 2024-11-27T13:40:43,435 INFO [StoreOpener-8700ed52628a2e71e47ec460999f2612-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8700ed52628a2e71e47ec460999f2612 columnFamilyName family 2024-11-27T13:40:43,435 DEBUG [StoreOpener-8700ed52628a2e71e47ec460999f2612-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T13:40:43,435 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for test,hhh,1732714842283.9ee34ecbedcb7dfdabcb99a67ac21045. 2024-11-27T13:40:43,435 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened test,hhh,1732714842283.9ee34ecbedcb7dfdabcb99a67ac21045. 2024-11-27T13:40:43,435 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(132): Open test,vvv,1732714842283.c4f747499d4a9438c3cddbfb1ead0d2b. 2024-11-27T13:40:43,435 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7752): Opening region: {ENCODED => c4f747499d4a9438c3cddbfb1ead0d2b, NAME => 'test,vvv,1732714842283.c4f747499d4a9438c3cddbfb1ead0d2b.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-27T13:40:43,436 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c4f747499d4a9438c3cddbfb1ead0d2b 2024-11-27T13:40:43,436 INFO [StoreOpener-8700ed52628a2e71e47ec460999f2612-1 {}] regionserver.HStore(327): Store=8700ed52628a2e71e47ec460999f2612/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T13:40:43,436 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(898): Instantiated test,vvv,1732714842283.c4f747499d4a9438c3cddbfb1ead0d2b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T13:40:43,436 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7794): checking encryption for c4f747499d4a9438c3cddbfb1ead0d2b 2024-11-27T13:40:43,436 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for 8700ed52628a2e71e47ec460999f2612 2024-11-27T13:40:43,436 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7797): checking classloading for c4f747499d4a9438c3cddbfb1ead0d2b 2024-11-27T13:40:43,436 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=23 2024-11-27T13:40:43,437 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure f5f0e25bfe9d22339132aab7b2f16f09, server=29b1b1ba230f,39411,1732714839711 in 294 msec 2024-11-27T13:40:43,437 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/8700ed52628a2e71e47ec460999f2612 2024-11-27T13:40:43,437 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/6b448045aa14a354ef4bc74b08978138/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T13:40:43,438 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/8700ed52628a2e71e47ec460999f2612 2024-11-27T13:40:43,438 INFO [StoreOpener-c4f747499d4a9438c3cddbfb1ead0d2b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c4f747499d4a9438c3cddbfb1ead0d2b 2024-11-27T13:40:43,438 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1114): Opened 6b448045aa14a354ef4bc74b08978138; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68803006, jitterRate=0.025244683027267456}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T13:40:43,439 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6b448045aa14a354ef4bc74b08978138 2024-11-27T13:40:43,439 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1006): Region open journal for 6b448045aa14a354ef4bc74b08978138: Running coprocessor pre-open hook at 1732714843416Writing region info on filesystem at 1732714843416Initializing all the Stores at 1732714843418 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732714843418Cleaning up temporary data from old regions at 1732714843428 (+10 ms)Running coprocessor post-open hooks at 1732714843439 (+11 ms)Region opened successfully at 1732714843439 2024-11-27T13:40:43,439 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for 8700ed52628a2e71e47ec460999f2612 2024-11-27T13:40:43,440 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for 8700ed52628a2e71e47ec460999f2612 2024-11-27T13:40:43,440 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2236): Post open deploy tasks for test,qqq,1732714842283.6b448045aa14a354ef4bc74b08978138., pid=51, masterSystemTime=1732714843300 2024-11-27T13:40:43,441 INFO [StoreOpener-c4f747499d4a9438c3cddbfb1ead0d2b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c4f747499d4a9438c3cddbfb1ead0d2b columnFamilyName family 2024-11-27T13:40:43,441 DEBUG [StoreOpener-c4f747499d4a9438c3cddbfb1ead0d2b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T13:40:43,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T13:40:43,442 INFO [StoreOpener-c4f747499d4a9438c3cddbfb1ead0d2b-1 {}] regionserver.HStore(327): Store=c4f747499d4a9438c3cddbfb1ead0d2b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T13:40:43,443 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for 8700ed52628a2e71e47ec460999f2612 2024-11-27T13:40:43,443 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1038): replaying wal for c4f747499d4a9438c3cddbfb1ead0d2b 2024-11-27T13:40:43,443 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f5f0e25bfe9d22339132aab7b2f16f09, ASSIGN in 544 msec 2024-11-27T13:40:43,444 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/c4f747499d4a9438c3cddbfb1ead0d2b 2024-11-27T13:40:43,445 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/c4f747499d4a9438c3cddbfb1ead0d2b 2024-11-27T13:40:43,445 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=9ee34ecbedcb7dfdabcb99a67ac21045, regionState=OPEN, openSeqNum=2, regionLocation=29b1b1ba230f,39411,1732714839711 2024-11-27T13:40:43,445 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2266): Finished post open deploy task for test,qqq,1732714842283.6b448045aa14a354ef4bc74b08978138. 2024-11-27T13:40:43,445 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(153): Opened test,qqq,1732714842283.6b448045aa14a354ef4bc74b08978138. 2024-11-27T13:40:43,445 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open test,zzz,1732714842283.6d41565a29ddd5c9153a0f41aba598ca. 2024-11-27T13:40:43,446 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1048): stopping wal replay for c4f747499d4a9438c3cddbfb1ead0d2b 2024-11-27T13:40:43,446 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1060): Cleaning up temporary data for c4f747499d4a9438c3cddbfb1ead0d2b 2024-11-27T13:40:43,446 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => 6d41565a29ddd5c9153a0f41aba598ca, NAME => 'test,zzz,1732714842283.6d41565a29ddd5c9153a0f41aba598ca.', STARTKEY => 'zzz', ENDKEY => ''} 2024-11-27T13:40:43,446 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 6d41565a29ddd5c9153a0f41aba598ca 2024-11-27T13:40:43,446 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated test,zzz,1732714842283.6d41565a29ddd5c9153a0f41aba598ca.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T13:40:43,446 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for 6d41565a29ddd5c9153a0f41aba598ca 2024-11-27T13:40:43,446 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for 6d41565a29ddd5c9153a0f41aba598ca 2024-11-27T13:40:43,450 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9ee34ecbedcb7dfdabcb99a67ac21045, server=29b1b1ba230f,39411,1732714839711 because future has completed 2024-11-27T13:40:43,450 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1093): writing seq id for c4f747499d4a9438c3cddbfb1ead0d2b 2024-11-27T13:40:43,451 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=6b448045aa14a354ef4bc74b08978138, regionState=OPEN, openSeqNum=2, regionLocation=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:43,455 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6b448045aa14a354ef4bc74b08978138, server=29b1b1ba230f,38845,1732714839599 because future has completed 2024-11-27T13:40:43,457 INFO [StoreOpener-6d41565a29ddd5c9153a0f41aba598ca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 6d41565a29ddd5c9153a0f41aba598ca 2024-11-27T13:40:43,457 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/8700ed52628a2e71e47ec460999f2612/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T13:40:43,458 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/c4f747499d4a9438c3cddbfb1ead0d2b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T13:40:43,459 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened 8700ed52628a2e71e47ec460999f2612; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69352109, jitterRate=0.033426955342292786}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T13:40:43,459 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8700ed52628a2e71e47ec460999f2612 2024-11-27T13:40:43,459 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1114): Opened c4f747499d4a9438c3cddbfb1ead0d2b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60309247, jitterRate=-0.10132218897342682}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T13:40:43,459 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=12 2024-11-27T13:40:43,459 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c4f747499d4a9438c3cddbfb1ead0d2b 2024-11-27T13:40:43,459 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure 9ee34ecbedcb7dfdabcb99a67ac21045, server=29b1b1ba230f,39411,1732714839711 in 285 msec 2024-11-27T13:40:43,459 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for 8700ed52628a2e71e47ec460999f2612: Running coprocessor pre-open hook at 1732714843429Writing region info on filesystem at 1732714843429Initializing all the Stores at 1732714843430 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732714843430Cleaning up temporary data from old regions at 1732714843440 (+10 ms)Running coprocessor post-open hooks at 1732714843459 (+19 ms)Region opened successfully at 1732714843459 2024-11-27T13:40:43,459 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1006): Region open journal for c4f747499d4a9438c3cddbfb1ead0d2b: Running coprocessor pre-open hook at 1732714843436Writing region info on filesystem at 1732714843436Initializing all the Stores at 1732714843438 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732714843438Cleaning up temporary data from old regions at 1732714843446 (+8 ms)Running coprocessor post-open hooks at 1732714843459 (+13 ms)Region opened successfully at 1732714843459 2024-11-27T13:40:43,460 INFO [StoreOpener-6d41565a29ddd5c9153a0f41aba598ca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6d41565a29ddd5c9153a0f41aba598ca columnFamilyName family 2024-11-27T13:40:43,460 DEBUG [StoreOpener-6d41565a29ddd5c9153a0f41aba598ca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T13:40:43,461 INFO [StoreOpener-6d41565a29ddd5c9153a0f41aba598ca-1 {}] regionserver.HStore(327): Store=6d41565a29ddd5c9153a0f41aba598ca/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T13:40:43,461 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1732714842283.8700ed52628a2e71e47ec460999f2612., pid=36, masterSystemTime=1732714843225 2024-11-27T13:40:43,461 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1732714842283.c4f747499d4a9438c3cddbfb1ead0d2b., pid=48, masterSystemTime=1732714843288 2024-11-27T13:40:43,462 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for 6d41565a29ddd5c9153a0f41aba598ca 2024-11-27T13:40:43,463 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/6d41565a29ddd5c9153a0f41aba598ca 2024-11-27T13:40:43,464 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN in 566 msec 2024-11-27T13:40:43,464 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/6d41565a29ddd5c9153a0f41aba598ca 2024-11-27T13:40:43,464 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1732714842283.8700ed52628a2e71e47ec460999f2612. 2024-11-27T13:40:43,464 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened test,rrr,1732714842283.8700ed52628a2e71e47ec460999f2612. 2024-11-27T13:40:43,464 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for 6d41565a29ddd5c9153a0f41aba598ca 2024-11-27T13:40:43,465 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for 6d41565a29ddd5c9153a0f41aba598ca 2024-11-27T13:40:43,465 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open test,uuu,1732714842283.457728e6ba667138202e8a0dfa04b5b7. 2024-11-27T13:40:43,465 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 457728e6ba667138202e8a0dfa04b5b7, NAME => 'test,uuu,1732714842283.457728e6ba667138202e8a0dfa04b5b7.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-27T13:40:43,465 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 457728e6ba667138202e8a0dfa04b5b7 2024-11-27T13:40:43,466 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated test,uuu,1732714842283.457728e6ba667138202e8a0dfa04b5b7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T13:40:43,466 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 457728e6ba667138202e8a0dfa04b5b7 2024-11-27T13:40:43,466 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 457728e6ba667138202e8a0dfa04b5b7 2024-11-27T13:40:43,468 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=8700ed52628a2e71e47ec460999f2612, regionState=OPEN, openSeqNum=2, regionLocation=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:43,468 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1732714842283.c4f747499d4a9438c3cddbfb1ead0d2b. 2024-11-27T13:40:43,468 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(153): Opened test,vvv,1732714842283.c4f747499d4a9438c3cddbfb1ead0d2b. 2024-11-27T13:40:43,468 INFO [StoreOpener-457728e6ba667138202e8a0dfa04b5b7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 457728e6ba667138202e8a0dfa04b5b7 2024-11-27T13:40:43,468 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open test,ppp,1732714842283.de4deb0c16ab89c9c56fc8912227f1c5. 2024-11-27T13:40:43,469 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => de4deb0c16ab89c9c56fc8912227f1c5, NAME => 'test,ppp,1732714842283.de4deb0c16ab89c9c56fc8912227f1c5.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-27T13:40:43,469 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test de4deb0c16ab89c9c56fc8912227f1c5 2024-11-27T13:40:43,469 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated test,ppp,1732714842283.de4deb0c16ab89c9c56fc8912227f1c5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T13:40:43,469 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for 6d41565a29ddd5c9153a0f41aba598ca 2024-11-27T13:40:43,469 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for de4deb0c16ab89c9c56fc8912227f1c5 2024-11-27T13:40:43,469 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for de4deb0c16ab89c9c56fc8912227f1c5 2024-11-27T13:40:43,471 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8700ed52628a2e71e47ec460999f2612, server=29b1b1ba230f,44709,1732714839765 because future has completed 2024-11-27T13:40:43,473 INFO [StoreOpener-de4deb0c16ab89c9c56fc8912227f1c5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region de4deb0c16ab89c9c56fc8912227f1c5 2024-11-27T13:40:43,473 INFO [StoreOpener-457728e6ba667138202e8a0dfa04b5b7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 457728e6ba667138202e8a0dfa04b5b7 columnFamilyName family 2024-11-27T13:40:43,473 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=c4f747499d4a9438c3cddbfb1ead0d2b, regionState=OPEN, openSeqNum=2, regionLocation=29b1b1ba230f,39411,1732714839711 2024-11-27T13:40:43,473 DEBUG [StoreOpener-457728e6ba667138202e8a0dfa04b5b7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T13:40:43,474 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/6d41565a29ddd5c9153a0f41aba598ca/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T13:40:43,475 INFO [StoreOpener-de4deb0c16ab89c9c56fc8912227f1c5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de4deb0c16ab89c9c56fc8912227f1c5 columnFamilyName family 2024-11-27T13:40:43,476 DEBUG [StoreOpener-de4deb0c16ab89c9c56fc8912227f1c5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T13:40:43,476 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened 6d41565a29ddd5c9153a0f41aba598ca; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71007233, jitterRate=0.058090224862098694}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T13:40:43,476 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6d41565a29ddd5c9153a0f41aba598ca 2024-11-27T13:40:43,476 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for 6d41565a29ddd5c9153a0f41aba598ca: Running coprocessor pre-open hook at 1732714843447Writing region info on filesystem at 1732714843447Initializing all the Stores at 1732714843449 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732714843449Cleaning up temporary data from old regions at 1732714843465 (+16 ms)Running coprocessor post-open hooks at 1732714843476 (+11 ms)Region opened successfully at 1732714843476 2024-11-27T13:40:43,476 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=48, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure c4f747499d4a9438c3cddbfb1ead0d2b, server=29b1b1ba230f,39411,1732714839711 because future has completed 2024-11-27T13:40:43,476 INFO [StoreOpener-de4deb0c16ab89c9c56fc8912227f1c5-1 {}] regionserver.HStore(327): Store=de4deb0c16ab89c9c56fc8912227f1c5/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T13:40:43,477 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for de4deb0c16ab89c9c56fc8912227f1c5 2024-11-27T13:40:43,478 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for test,zzz,1732714842283.6d41565a29ddd5c9153a0f41aba598ca., pid=41, masterSystemTime=1732714843300 2024-11-27T13:40:43,478 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/de4deb0c16ab89c9c56fc8912227f1c5 2024-11-27T13:40:43,478 INFO [StoreOpener-457728e6ba667138202e8a0dfa04b5b7-1 {}] regionserver.HStore(327): Store=457728e6ba667138202e8a0dfa04b5b7/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T13:40:43,478 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 457728e6ba667138202e8a0dfa04b5b7 2024-11-27T13:40:43,479 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/de4deb0c16ab89c9c56fc8912227f1c5 2024-11-27T13:40:43,480 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for de4deb0c16ab89c9c56fc8912227f1c5 2024-11-27T13:40:43,480 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/457728e6ba667138202e8a0dfa04b5b7 2024-11-27T13:40:43,480 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for de4deb0c16ab89c9c56fc8912227f1c5 2024-11-27T13:40:43,481 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/457728e6ba667138202e8a0dfa04b5b7 2024-11-27T13:40:43,482 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for test,zzz,1732714842283.6d41565a29ddd5c9153a0f41aba598ca. 2024-11-27T13:40:43,482 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 457728e6ba667138202e8a0dfa04b5b7 2024-11-27T13:40:43,482 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 457728e6ba667138202e8a0dfa04b5b7 2024-11-27T13:40:43,482 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened test,zzz,1732714842283.6d41565a29ddd5c9153a0f41aba598ca. 2024-11-27T13:40:43,482 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(132): Open test,ttt,1732714842283.922622f55d3d471d091912fe46d7c8d8. 2024-11-27T13:40:43,484 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for de4deb0c16ab89c9c56fc8912227f1c5 2024-11-27T13:40:43,483 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7752): Opening region: {ENCODED => 922622f55d3d471d091912fe46d7c8d8, NAME => 'test,ttt,1732714842283.922622f55d3d471d091912fe46d7c8d8.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-27T13:40:43,484 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 922622f55d3d471d091912fe46d7c8d8 2024-11-27T13:40:43,484 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(898): Instantiated test,ttt,1732714842283.922622f55d3d471d091912fe46d7c8d8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T13:40:43,484 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7794): checking encryption for 922622f55d3d471d091912fe46d7c8d8 2024-11-27T13:40:43,484 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7797): checking classloading for 922622f55d3d471d091912fe46d7c8d8 2024-11-27T13:40:43,485 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=22 2024-11-27T13:40:43,485 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure 8700ed52628a2e71e47ec460999f2612, server=29b1b1ba230f,44709,1732714839765 in 374 msec 2024-11-27T13:40:43,485 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 457728e6ba667138202e8a0dfa04b5b7 2024-11-27T13:40:43,487 INFO [StoreOpener-922622f55d3d471d091912fe46d7c8d8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 922622f55d3d471d091912fe46d7c8d8 2024-11-27T13:40:43,489 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/de4deb0c16ab89c9c56fc8912227f1c5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T13:40:43,489 INFO [StoreOpener-922622f55d3d471d091912fe46d7c8d8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 922622f55d3d471d091912fe46d7c8d8 columnFamilyName family 2024-11-27T13:40:43,490 DEBUG [StoreOpener-922622f55d3d471d091912fe46d7c8d8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T13:40:43,490 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8700ed52628a2e71e47ec460999f2612, ASSIGN in 592 msec 2024-11-27T13:40:43,490 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened de4deb0c16ab89c9c56fc8912227f1c5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75380127, jitterRate=0.12325142323970795}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T13:40:43,491 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for de4deb0c16ab89c9c56fc8912227f1c5 2024-11-27T13:40:43,491 INFO [StoreOpener-922622f55d3d471d091912fe46d7c8d8-1 {}] regionserver.HStore(327): Store=922622f55d3d471d091912fe46d7c8d8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T13:40:43,491 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for de4deb0c16ab89c9c56fc8912227f1c5: Running coprocessor pre-open hook at 1732714843470Writing region info on filesystem at 1732714843470Initializing all the Stores at 1732714843472 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732714843472Cleaning up temporary data from old regions at 1732714843480 (+8 ms)Running coprocessor post-open hooks at 1732714843491 (+11 ms)Region opened successfully at 1732714843491 2024-11-27T13:40:43,491 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1038): replaying wal for 922622f55d3d471d091912fe46d7c8d8 2024-11-27T13:40:43,491 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/457728e6ba667138202e8a0dfa04b5b7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T13:40:43,492 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 457728e6ba667138202e8a0dfa04b5b7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69072077, jitterRate=0.02925415337085724}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T13:40:43,492 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 457728e6ba667138202e8a0dfa04b5b7 2024-11-27T13:40:43,492 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/922622f55d3d471d091912fe46d7c8d8 2024-11-27T13:40:43,492 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 457728e6ba667138202e8a0dfa04b5b7: Running coprocessor pre-open hook at 1732714843466Writing region info on filesystem at 1732714843466Initializing all the Stores at 1732714843468 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732714843468Cleaning up temporary data from old regions at 1732714843482 (+14 ms)Running coprocessor post-open hooks at 1732714843492 (+10 ms)Region opened successfully at 1732714843492 2024-11-27T13:40:43,492 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1732714842283.de4deb0c16ab89c9c56fc8912227f1c5., pid=47, masterSystemTime=1732714843288 2024-11-27T13:40:43,493 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=6d41565a29ddd5c9153a0f41aba598ca, regionState=OPEN, openSeqNum=2, regionLocation=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:43,493 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/922622f55d3d471d091912fe46d7c8d8 2024-11-27T13:40:43,493 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for test,uuu,1732714842283.457728e6ba667138202e8a0dfa04b5b7., pid=37, masterSystemTime=1732714843225 2024-11-27T13:40:43,494 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1048): stopping wal replay for 922622f55d3d471d091912fe46d7c8d8 2024-11-27T13:40:43,494 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1060): Cleaning up temporary data for 922622f55d3d471d091912fe46d7c8d8 2024-11-27T13:40:43,496 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6d41565a29ddd5c9153a0f41aba598ca, server=29b1b1ba230f,38845,1732714839599 because future has completed 2024-11-27T13:40:43,497 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ppp,1732714842283.de4deb0c16ab89c9c56fc8912227f1c5. 2024-11-27T13:40:43,497 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened test,ppp,1732714842283.de4deb0c16ab89c9c56fc8912227f1c5. 2024-11-27T13:40:43,497 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(132): Open test,yyy,1732714842283.d64103cc558d5243fdfd1e469558a599. 2024-11-27T13:40:43,497 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7752): Opening region: {ENCODED => d64103cc558d5243fdfd1e469558a599, NAME => 'test,yyy,1732714842283.d64103cc558d5243fdfd1e469558a599.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-27T13:40:43,498 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d64103cc558d5243fdfd1e469558a599 2024-11-27T13:40:43,498 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(898): Instantiated test,yyy,1732714842283.d64103cc558d5243fdfd1e469558a599.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T13:40:43,498 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7794): checking encryption for d64103cc558d5243fdfd1e469558a599 2024-11-27T13:40:43,498 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7797): checking classloading for d64103cc558d5243fdfd1e469558a599 2024-11-27T13:40:43,499 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1093): writing seq id for 922622f55d3d471d091912fe46d7c8d8 2024-11-27T13:40:43,500 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for test,uuu,1732714842283.457728e6ba667138202e8a0dfa04b5b7. 2024-11-27T13:40:43,500 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=de4deb0c16ab89c9c56fc8912227f1c5, regionState=OPEN, openSeqNum=2, regionLocation=29b1b1ba230f,39411,1732714839711 2024-11-27T13:40:43,500 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened test,uuu,1732714842283.457728e6ba667138202e8a0dfa04b5b7. 2024-11-27T13:40:43,501 INFO [StoreOpener-d64103cc558d5243fdfd1e469558a599-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d64103cc558d5243fdfd1e469558a599 2024-11-27T13:40:43,503 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/922622f55d3d471d091912fe46d7c8d8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T13:40:43,504 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1114): Opened 922622f55d3d471d091912fe46d7c8d8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74896070, jitterRate=0.11603841185569763}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T13:40:43,504 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 922622f55d3d471d091912fe46d7c8d8 2024-11-27T13:40:43,504 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,39411,1732714839711 because future has completed 2024-11-27T13:40:43,504 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1006): Region open journal for 922622f55d3d471d091912fe46d7c8d8: Running coprocessor pre-open hook at 1732714843485Writing region info on filesystem at 1732714843485Initializing all the Stores at 1732714843486 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732714843486Cleaning up temporary data from old regions at 1732714843494 (+8 ms)Running coprocessor post-open hooks at 1732714843504 (+10 ms)Region opened successfully at 1732714843504 2024-11-27T13:40:43,504 INFO [StoreOpener-d64103cc558d5243fdfd1e469558a599-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d64103cc558d5243fdfd1e469558a599 columnFamilyName family 2024-11-27T13:40:43,504 DEBUG [StoreOpener-d64103cc558d5243fdfd1e469558a599-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T13:40:43,505 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1732714842283.922622f55d3d471d091912fe46d7c8d8., pid=42, masterSystemTime=1732714843300 2024-11-27T13:40:43,505 INFO [StoreOpener-d64103cc558d5243fdfd1e469558a599-1 {}] regionserver.HStore(327): Store=d64103cc558d5243fdfd1e469558a599/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T13:40:43,506 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1038): replaying wal for d64103cc558d5243fdfd1e469558a599 2024-11-27T13:40:43,507 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/d64103cc558d5243fdfd1e469558a599 2024-11-27T13:40:43,508 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/d64103cc558d5243fdfd1e469558a599 2024-11-27T13:40:43,508 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1732714842283.922622f55d3d471d091912fe46d7c8d8. 2024-11-27T13:40:43,508 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(153): Opened test,ttt,1732714842283.922622f55d3d471d091912fe46d7c8d8. 2024-11-27T13:40:43,508 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(132): Open test,ggg,1732714842283.85dc78011fc464d1446075ad14a217de. 2024-11-27T13:40:43,509 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7752): Opening region: {ENCODED => 85dc78011fc464d1446075ad14a217de, NAME => 'test,ggg,1732714842283.85dc78011fc464d1446075ad14a217de.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-11-27T13:40:43,509 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 85dc78011fc464d1446075ad14a217de 2024-11-27T13:40:43,509 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(898): Instantiated test,ggg,1732714842283.85dc78011fc464d1446075ad14a217de.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T13:40:43,509 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1048): stopping wal replay for d64103cc558d5243fdfd1e469558a599 2024-11-27T13:40:43,509 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1060): Cleaning up temporary data for d64103cc558d5243fdfd1e469558a599 2024-11-27T13:40:43,509 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7794): checking encryption for 85dc78011fc464d1446075ad14a217de 2024-11-27T13:40:43,509 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7797): checking classloading for 85dc78011fc464d1446075ad14a217de 2024-11-27T13:40:43,512 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): writing seq id for d64103cc558d5243fdfd1e469558a599 2024-11-27T13:40:43,513 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=922622f55d3d471d091912fe46d7c8d8, regionState=OPEN, openSeqNum=2, regionLocation=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:43,517 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 922622f55d3d471d091912fe46d7c8d8, server=29b1b1ba230f,38845,1732714839599 because future has completed 2024-11-27T13:40:43,521 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=20 2024-11-27T13:40:43,521 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,39411,1732714839711 in 338 msec 2024-11-27T13:40:43,523 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=de4deb0c16ab89c9c56fc8912227f1c5, ASSIGN in 628 msec 2024-11-27T13:40:43,526 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/d64103cc558d5243fdfd1e469558a599/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T13:40:43,527 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1114): Opened d64103cc558d5243fdfd1e469558a599; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65132233, jitterRate=-0.029454097151756287}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T13:40:43,527 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d64103cc558d5243fdfd1e469558a599 2024-11-27T13:40:43,527 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1006): Region open journal for d64103cc558d5243fdfd1e469558a599: Running coprocessor pre-open hook at 1732714843498Writing region info on filesystem at 1732714843498Initializing all the Stores at 1732714843500 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732714843500Cleaning up temporary data from old regions at 1732714843509 (+9 ms)Running coprocessor post-open hooks at 1732714843527 (+18 ms)Region opened successfully at 1732714843527 2024-11-27T13:40:43,529 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1732714842283.d64103cc558d5243fdfd1e469558a599., pid=49, masterSystemTime=1732714843288 2024-11-27T13:40:43,530 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35019 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=29b1b1ba230f,38845,1732714839599, table=test, region=4d1ad27263331aa341edfa82a041de56. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-27T13:40:43,530 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35019 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=29b1b1ba230f,38845,1732714839599, table=test, region=6b448045aa14a354ef4bc74b08978138. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-27T13:40:43,530 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35019 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=29b1b1ba230f,38845,1732714839599, table=test, region=6d41565a29ddd5c9153a0f41aba598ca. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-27T13:40:43,534 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1732714842283.d64103cc558d5243fdfd1e469558a599. 2024-11-27T13:40:43,534 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=24 2024-11-27T13:40:43,534 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(153): Opened test,yyy,1732714842283.d64103cc558d5243fdfd1e469558a599. 2024-11-27T13:40:43,534 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure 922622f55d3d471d091912fe46d7c8d8, server=29b1b1ba230f,38845,1732714839599 in 371 msec 2024-11-27T13:40:43,534 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open test,mmm,1732714842283.cccf7ce6a567350b0ae162454b0a9379. 2024-11-27T13:40:43,535 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => cccf7ce6a567350b0ae162454b0a9379, NAME => 'test,mmm,1732714842283.cccf7ce6a567350b0ae162454b0a9379.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-27T13:40:43,535 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test cccf7ce6a567350b0ae162454b0a9379 2024-11-27T13:40:43,535 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated test,mmm,1732714842283.cccf7ce6a567350b0ae162454b0a9379.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T13:40:43,535 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for cccf7ce6a567350b0ae162454b0a9379 2024-11-27T13:40:43,536 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for cccf7ce6a567350b0ae162454b0a9379 2024-11-27T13:40:43,536 INFO [StoreOpener-85dc78011fc464d1446075ad14a217de-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 85dc78011fc464d1446075ad14a217de 2024-11-27T13:40:43,538 INFO [StoreOpener-85dc78011fc464d1446075ad14a217de-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 85dc78011fc464d1446075ad14a217de columnFamilyName family 2024-11-27T13:40:43,538 DEBUG [StoreOpener-85dc78011fc464d1446075ad14a217de-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T13:40:43,539 INFO [StoreOpener-85dc78011fc464d1446075ad14a217de-1 {}] regionserver.HStore(327): Store=85dc78011fc464d1446075ad14a217de/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T13:40:43,539 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1038): replaying wal for 85dc78011fc464d1446075ad14a217de 2024-11-27T13:40:43,540 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35019 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=29b1b1ba230f,39411,1732714839711, table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-27T13:40:43,540 INFO [StoreOpener-cccf7ce6a567350b0ae162454b0a9379-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region cccf7ce6a567350b0ae162454b0a9379 2024-11-27T13:40:43,541 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=922622f55d3d471d091912fe46d7c8d8, ASSIGN in 641 msec 2024-11-27T13:40:43,541 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/85dc78011fc464d1446075ad14a217de 2024-11-27T13:40:43,541 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/85dc78011fc464d1446075ad14a217de 2024-11-27T13:40:43,542 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1048): stopping wal replay for 85dc78011fc464d1446075ad14a217de 2024-11-27T13:40:43,542 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1060): Cleaning up temporary data for 85dc78011fc464d1446075ad14a217de 2024-11-27T13:40:43,544 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=d64103cc558d5243fdfd1e469558a599, regionState=OPEN, openSeqNum=2, regionLocation=29b1b1ba230f,39411,1732714839711 2024-11-27T13:40:43,546 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1093): writing seq id for 85dc78011fc464d1446075ad14a217de 2024-11-27T13:40:43,547 INFO [StoreOpener-cccf7ce6a567350b0ae162454b0a9379-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cccf7ce6a567350b0ae162454b0a9379 columnFamilyName family 2024-11-27T13:40:43,547 DEBUG [StoreOpener-cccf7ce6a567350b0ae162454b0a9379-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T13:40:43,548 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure d64103cc558d5243fdfd1e469558a599, server=29b1b1ba230f,39411,1732714839711 because future has completed 2024-11-27T13:40:43,549 INFO [StoreOpener-cccf7ce6a567350b0ae162454b0a9379-1 {}] regionserver.HStore(327): Store=cccf7ce6a567350b0ae162454b0a9379/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T13:40:43,549 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for cccf7ce6a567350b0ae162454b0a9379 2024-11-27T13:40:43,550 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=457728e6ba667138202e8a0dfa04b5b7, regionState=OPEN, openSeqNum=2, regionLocation=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:43,550 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/85dc78011fc464d1446075ad14a217de/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T13:40:43,550 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/cccf7ce6a567350b0ae162454b0a9379 2024-11-27T13:40:43,551 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1114): Opened 85dc78011fc464d1446075ad14a217de; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65087268, jitterRate=-0.030124127864837646}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T13:40:43,551 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 85dc78011fc464d1446075ad14a217de 2024-11-27T13:40:43,551 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/cccf7ce6a567350b0ae162454b0a9379 2024-11-27T13:40:43,551 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1006): Region open journal for 85dc78011fc464d1446075ad14a217de: Running coprocessor pre-open hook at 1732714843509Writing region info on filesystem at 1732714843509Initializing all the Stores at 1732714843511 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732714843511Cleaning up temporary data from old regions at 1732714843542 (+31 ms)Running coprocessor post-open hooks at 1732714843551 (+9 ms)Region opened successfully at 1732714843551 2024-11-27T13:40:43,552 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ggg,1732714842283.85dc78011fc464d1446075ad14a217de., pid=54, masterSystemTime=1732714843300 2024-11-27T13:40:43,552 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for cccf7ce6a567350b0ae162454b0a9379 2024-11-27T13:40:43,552 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for cccf7ce6a567350b0ae162454b0a9379 2024-11-27T13:40:43,553 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 457728e6ba667138202e8a0dfa04b5b7, server=29b1b1ba230f,44709,1732714839765 because future has completed 2024-11-27T13:40:43,556 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ggg,1732714842283.85dc78011fc464d1446075ad14a217de. 2024-11-27T13:40:43,556 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for cccf7ce6a567350b0ae162454b0a9379 2024-11-27T13:40:43,556 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(153): Opened test,ggg,1732714842283.85dc78011fc464d1446075ad14a217de. 2024-11-27T13:40:43,556 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(132): Open test,nnn,1732714842283.d095bb01efaabc5bf5541d373d5419fa. 2024-11-27T13:40:43,557 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7752): Opening region: {ENCODED => d095bb01efaabc5bf5541d373d5419fa, NAME => 'test,nnn,1732714842283.d095bb01efaabc5bf5541d373d5419fa.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-11-27T13:40:43,557 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d095bb01efaabc5bf5541d373d5419fa 2024-11-27T13:40:43,557 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(898): Instantiated test,nnn,1732714842283.d095bb01efaabc5bf5541d373d5419fa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T13:40:43,558 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7794): checking encryption for d095bb01efaabc5bf5541d373d5419fa 2024-11-27T13:40:43,558 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=29 2024-11-27T13:40:43,558 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7797): checking classloading for d095bb01efaabc5bf5541d373d5419fa 2024-11-27T13:40:43,558 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure d64103cc558d5243fdfd1e469558a599, server=29b1b1ba230f,39411,1732714839711 in 367 msec 2024-11-27T13:40:43,561 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d64103cc558d5243fdfd1e469558a599, ASSIGN in 665 msec 2024-11-27T13:40:43,562 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=85dc78011fc464d1446075ad14a217de, regionState=OPEN, openSeqNum=2, regionLocation=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:43,565 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 85dc78011fc464d1446075ad14a217de, server=29b1b1ba230f,38845,1732714839599 because future has completed 2024-11-27T13:40:43,569 INFO [StoreOpener-d095bb01efaabc5bf5541d373d5419fa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d095bb01efaabc5bf5541d373d5419fa 2024-11-27T13:40:43,570 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/cccf7ce6a567350b0ae162454b0a9379/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T13:40:43,570 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened cccf7ce6a567350b0ae162454b0a9379; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73976241, jitterRate=0.10233189165592194}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T13:40:43,570 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cccf7ce6a567350b0ae162454b0a9379 2024-11-27T13:40:43,571 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=25 2024-11-27T13:40:43,571 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for cccf7ce6a567350b0ae162454b0a9379: Running coprocessor pre-open hook at 1732714843536Writing region info on filesystem at 1732714843536Initializing all the Stores at 1732714843537 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732714843538 (+1 ms)Cleaning up temporary data from old regions at 1732714843552 (+14 ms)Running coprocessor post-open hooks at 1732714843570 (+18 ms)Region opened successfully at 1732714843570 2024-11-27T13:40:43,571 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure 457728e6ba667138202e8a0dfa04b5b7, server=29b1b1ba230f,44709,1732714839765 in 458 msec 2024-11-27T13:40:43,571 INFO [StoreOpener-d095bb01efaabc5bf5541d373d5419fa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d095bb01efaabc5bf5541d373d5419fa columnFamilyName family 2024-11-27T13:40:43,571 DEBUG [StoreOpener-d095bb01efaabc5bf5541d373d5419fa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T13:40:43,571 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1732714842283.cccf7ce6a567350b0ae162454b0a9379., pid=46, masterSystemTime=1732714843288 2024-11-27T13:40:43,572 INFO [StoreOpener-d095bb01efaabc5bf5541d373d5419fa-1 {}] regionserver.HStore(327): Store=d095bb01efaabc5bf5541d373d5419fa/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T13:40:43,572 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1038): replaying wal for d095bb01efaabc5bf5541d373d5419fa 2024-11-27T13:40:43,573 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/d095bb01efaabc5bf5541d373d5419fa 2024-11-27T13:40:43,574 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/d095bb01efaabc5bf5541d373d5419fa 2024-11-27T13:40:43,574 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=457728e6ba667138202e8a0dfa04b5b7, ASSIGN in 678 msec 2024-11-27T13:40:43,574 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1732714842283.cccf7ce6a567350b0ae162454b0a9379. 2024-11-27T13:40:43,574 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened test,mmm,1732714842283.cccf7ce6a567350b0ae162454b0a9379. 2024-11-27T13:40:43,574 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1048): stopping wal replay for d095bb01efaabc5bf5541d373d5419fa 2024-11-27T13:40:43,575 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1060): Cleaning up temporary data for d095bb01efaabc5bf5541d373d5419fa 2024-11-27T13:40:43,575 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=cccf7ce6a567350b0ae162454b0a9379, regionState=OPEN, openSeqNum=2, regionLocation=29b1b1ba230f,39411,1732714839711 2024-11-27T13:40:43,579 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure cccf7ce6a567350b0ae162454b0a9379, server=29b1b1ba230f,39411,1732714839711 because future has completed 2024-11-27T13:40:43,580 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1093): writing seq id for d095bb01efaabc5bf5541d373d5419fa 2024-11-27T13:40:43,583 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=11 2024-11-27T13:40:43,583 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 85dc78011fc464d1446075ad14a217de, server=29b1b1ba230f,38845,1732714839599 in 365 msec 2024-11-27T13:40:43,583 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/d095bb01efaabc5bf5541d373d5419fa/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T13:40:43,584 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1114): Opened d095bb01efaabc5bf5541d373d5419fa; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62902800, jitterRate=-0.06267523765563965}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T13:40:43,585 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d095bb01efaabc5bf5541d373d5419fa 2024-11-27T13:40:43,585 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1006): Region open journal for d095bb01efaabc5bf5541d373d5419fa: Running coprocessor pre-open hook at 1732714843558Writing region info on filesystem at 1732714843558Initializing all the Stores at 1732714843559 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732714843559Cleaning up temporary data from old regions at 1732714843575 (+16 ms)Running coprocessor post-open hooks at 1732714843585 (+10 ms)Region opened successfully at 1732714843585 2024-11-27T13:40:43,586 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2236): Post open deploy tasks for test,nnn,1732714842283.d095bb01efaabc5bf5541d373d5419fa., pid=56, masterSystemTime=1732714843300 2024-11-27T13:40:43,586 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=85dc78011fc464d1446075ad14a217de, ASSIGN in 690 msec 2024-11-27T13:40:43,588 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2266): Finished post open deploy task for test,nnn,1732714842283.d095bb01efaabc5bf5541d373d5419fa. 2024-11-27T13:40:43,588 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(153): Opened test,nnn,1732714842283.d095bb01efaabc5bf5541d373d5419fa. 2024-11-27T13:40:43,589 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open test,,1732714842283.51060408ec4ea38afe378df89549aeae. 2024-11-27T13:40:43,589 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => 51060408ec4ea38afe378df89549aeae, NAME => 'test,,1732714842283.51060408ec4ea38afe378df89549aeae.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-27T13:40:43,589 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 51060408ec4ea38afe378df89549aeae 2024-11-27T13:40:43,589 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated test,,1732714842283.51060408ec4ea38afe378df89549aeae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T13:40:43,589 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for 51060408ec4ea38afe378df89549aeae 2024-11-27T13:40:43,589 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for 51060408ec4ea38afe378df89549aeae 2024-11-27T13:40:43,591 INFO [StoreOpener-51060408ec4ea38afe378df89549aeae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 51060408ec4ea38afe378df89549aeae 2024-11-27T13:40:43,592 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=17 2024-11-27T13:40:43,592 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure cccf7ce6a567350b0ae162454b0a9379, server=29b1b1ba230f,39411,1732714839711 in 413 msec 2024-11-27T13:40:43,594 INFO [StoreOpener-51060408ec4ea38afe378df89549aeae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 51060408ec4ea38afe378df89549aeae columnFamilyName family 2024-11-27T13:40:43,594 DEBUG [StoreOpener-51060408ec4ea38afe378df89549aeae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T13:40:43,595 INFO [StoreOpener-51060408ec4ea38afe378df89549aeae-1 {}] regionserver.HStore(327): Store=51060408ec4ea38afe378df89549aeae/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T13:40:43,595 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=cccf7ce6a567350b0ae162454b0a9379, ASSIGN in 699 msec 2024-11-27T13:40:43,595 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for 51060408ec4ea38afe378df89549aeae 2024-11-27T13:40:43,597 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/51060408ec4ea38afe378df89549aeae 2024-11-27T13:40:43,597 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=d095bb01efaabc5bf5541d373d5419fa, regionState=OPEN, openSeqNum=2, regionLocation=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:43,597 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/51060408ec4ea38afe378df89549aeae 2024-11-27T13:40:43,598 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for 51060408ec4ea38afe378df89549aeae 2024-11-27T13:40:43,598 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for 51060408ec4ea38afe378df89549aeae 2024-11-27T13:40:43,601 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure d095bb01efaabc5bf5541d373d5419fa, server=29b1b1ba230f,38845,1732714839599 because future has completed 2024-11-27T13:40:43,603 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for 51060408ec4ea38afe378df89549aeae 2024-11-27T13:40:43,606 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=30 2024-11-27T13:40:43,606 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure 6d41565a29ddd5c9153a0f41aba598ca, server=29b1b1ba230f,38845,1732714839599 in 454 msec 2024-11-27T13:40:43,609 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6d41565a29ddd5c9153a0f41aba598ca, ASSIGN in 713 msec 2024-11-27T13:40:43,614 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=18 2024-11-27T13:40:43,615 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure d095bb01efaabc5bf5541d373d5419fa, server=29b1b1ba230f,38845,1732714839599 in 391 msec 2024-11-27T13:40:43,617 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d095bb01efaabc5bf5541d373d5419fa, ASSIGN in 722 msec 2024-11-27T13:40:43,622 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=26 2024-11-27T13:40:43,622 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure c4f747499d4a9438c3cddbfb1ead0d2b, server=29b1b1ba230f,39411,1732714839711 in 436 msec 2024-11-27T13:40:43,626 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN in 729 msec 2024-11-27T13:40:43,629 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/51060408ec4ea38afe378df89549aeae/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T13:40:43,630 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened 51060408ec4ea38afe378df89549aeae; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72670272, jitterRate=0.0828714370727539}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T13:40:43,631 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 51060408ec4ea38afe378df89549aeae 2024-11-27T13:40:43,631 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for 51060408ec4ea38afe378df89549aeae: Running coprocessor pre-open hook at 1732714843589Writing region info on filesystem at 1732714843589Initializing all the Stores at 1732714843591 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732714843591Cleaning up temporary data from old regions at 1732714843598 (+7 ms)Running coprocessor post-open hooks at 1732714843631 (+33 ms)Region opened successfully at 1732714843631 2024-11-27T13:40:43,634 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=21 2024-11-27T13:40:43,634 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35019 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=29b1b1ba230f,38845,1732714839599, table=test, region=4d1ad27263331aa341edfa82a041de56. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-27T13:40:43,634 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=21, state=SUCCESS, hasLock=false; OpenRegionProcedure 6b448045aa14a354ef4bc74b08978138, server=29b1b1ba230f,38845,1732714839599 in 429 msec 2024-11-27T13:40:43,634 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1732714842283.51060408ec4ea38afe378df89549aeae., pid=53, masterSystemTime=1732714843300 2024-11-27T13:40:43,638 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for test,,1732714842283.51060408ec4ea38afe378df89549aeae. 2024-11-27T13:40:43,638 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened test,,1732714842283.51060408ec4ea38afe378df89549aeae. 2024-11-27T13:40:43,639 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6b448045aa14a354ef4bc74b08978138, ASSIGN in 741 msec 2024-11-27T13:40:43,641 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=51060408ec4ea38afe378df89549aeae, regionState=OPEN, openSeqNum=2, regionLocation=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:43,645 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 51060408ec4ea38afe378df89549aeae, server=29b1b1ba230f,38845,1732714839599 because future has completed 2024-11-27T13:40:43,646 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=0aef6a7e4e138b1b9d7f4aea6971acc6, regionState=OPEN, openSeqNum=2, regionLocation=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:43,649 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0aef6a7e4e138b1b9d7f4aea6971acc6, server=29b1b1ba230f,44709,1732714839765 because future has completed 2024-11-27T13:40:43,655 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=5 2024-11-27T13:40:43,655 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 51060408ec4ea38afe378df89549aeae, server=29b1b1ba230f,38845,1732714839599 in 442 msec 2024-11-27T13:40:43,657 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=51060408ec4ea38afe378df89549aeae, ASSIGN in 762 msec 2024-11-27T13:40:43,662 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=7 2024-11-27T13:40:43,663 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 0aef6a7e4e138b1b9d7f4aea6971acc6, server=29b1b1ba230f,44709,1732714839765 in 468 msec 2024-11-27T13:40:43,665 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=0aef6a7e4e138b1b9d7f4aea6971acc6, ASSIGN in 770 msec 2024-11-27T13:40:43,669 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=9 2024-11-27T13:40:43,669 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure 4d1ad27263331aa341edfa82a041de56, server=29b1b1ba230f,38845,1732714839599 in 462 msec 2024-11-27T13:40:43,671 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=4d1ad27263331aa341edfa82a041de56, ASSIGN in 777 msec 2024-11-27T13:40:43,672 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=9c46624524e53a899138ac5a715a35d6, regionState=OPEN, openSeqNum=2, regionLocation=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:43,676 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9c46624524e53a899138ac5a715a35d6, server=29b1b1ba230f,44709,1732714839765 because future has completed 2024-11-27T13:40:43,676 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=ed774066130adf09502c61d07d72f5ab, regionState=OPEN, openSeqNum=2, regionLocation=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:43,679 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure ed774066130adf09502c61d07d72f5ab, server=29b1b1ba230f,38845,1732714839599 because future has completed 2024-11-27T13:40:43,682 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=19 2024-11-27T13:40:43,682 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=19, state=SUCCESS, hasLock=false; OpenRegionProcedure 9c46624524e53a899138ac5a715a35d6, server=29b1b1ba230f,44709,1732714839765 in 581 msec 2024-11-27T13:40:43,686 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=9c46624524e53a899138ac5a715a35d6, ASSIGN in 789 msec 2024-11-27T13:40:43,690 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=27 2024-11-27T13:40:43,691 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure ed774066130adf09502c61d07d72f5ab, server=29b1b1ba230f,38845,1732714839599 in 529 msec 2024-11-27T13:40:43,695 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=4 2024-11-27T13:40:43,695 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=ed774066130adf09502c61d07d72f5ab, ASSIGN in 798 msec 2024-11-27T13:40:43,697 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T13:40:43,697 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732714843697"}]},"ts":"1732714843697"} 2024-11-27T13:40:43,700 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLED in hbase:meta 2024-11-27T13:40:43,702 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T13:40:43,711 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test in 1.4110 sec 2024-11-27T13:40:44,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T13:40:44,464 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table test get assigned. Timeout = 60000ms 2024-11-27T13:40:44,464 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test completed 2024-11-27T13:40:44,466 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T13:40:44,490 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table test assigned to meta. Checking AM states. 2024-11-27T13:40:44,490 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T13:40:44,491 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table test assigned. 2024-11-27T13:40:44,492 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T13:40:44,497 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-27T13:40:44,517 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-27T13:40:44,532 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T13:40:44,535 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39676, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T13:40:44,539 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T13:40:44,541 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37704, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T13:40:44,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35019 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 51060408ec4ea38afe378df89549aeae changed from -1.0 to 0.0, refreshing cache 2024-11-27T13:40:44,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35019 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 4d1ad27263331aa341edfa82a041de56 changed from -1.0 to 0.0, refreshing cache 2024-11-27T13:40:44,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35019 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 85dc78011fc464d1446075ad14a217de changed from -1.0 to 0.0, refreshing cache 2024-11-27T13:40:44,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35019 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region eeadb5717c90d7f7a33c12d4883bf2b0 changed from -1.0 to 0.0, refreshing cache 2024-11-27T13:40:44,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35019 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region d095bb01efaabc5bf5541d373d5419fa changed from -1.0 to 0.0, refreshing cache 2024-11-27T13:40:44,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35019 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 6b448045aa14a354ef4bc74b08978138 changed from -1.0 to 0.0, refreshing cache 2024-11-27T13:40:44,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35019 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 922622f55d3d471d091912fe46d7c8d8 changed from -1.0 to 0.0, refreshing cache 2024-11-27T13:40:44,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35019 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region ed774066130adf09502c61d07d72f5ab changed from -1.0 to 0.0, refreshing cache 2024-11-27T13:40:44,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35019 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 6d41565a29ddd5c9153a0f41aba598ca changed from -1.0 to 0.0, refreshing cache 2024-11-27T13:40:44,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35019 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 2b6b183b9bc56b2bc4dc6f8968ee01c8 changed from -1.0 to 0.0, refreshing cache 2024-11-27T13:40:44,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35019 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 9ee34ecbedcb7dfdabcb99a67ac21045 changed from -1.0 to 0.0, refreshing cache 2024-11-27T13:40:44,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35019 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region e7a7c9c02e40375cd974ffe51b6da75b changed from -1.0 to 0.0, refreshing cache 2024-11-27T13:40:44,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35019 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region cccf7ce6a567350b0ae162454b0a9379 changed from -1.0 to 0.0, refreshing cache 2024-11-27T13:40:44,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35019 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region de4deb0c16ab89c9c56fc8912227f1c5 changed from -1.0 to 0.0, refreshing cache 2024-11-27T13:40:44,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35019 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f5f0e25bfe9d22339132aab7b2f16f09 changed from -1.0 to 0.0, refreshing cache 2024-11-27T13:40:44,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35019 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c4f747499d4a9438c3cddbfb1ead0d2b changed from -1.0 to 0.0, refreshing cache 2024-11-27T13:40:44,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35019 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region d64103cc558d5243fdfd1e469558a599 changed from -1.0 to 0.0, refreshing cache 2024-11-27T13:40:44,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35019 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 0aef6a7e4e138b1b9d7f4aea6971acc6 changed from -1.0 to 0.0, refreshing cache 2024-11-27T13:40:44,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35019 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 49523b5c318014c3e33791ea6f5e308d changed from -1.0 to 0.0, refreshing cache 2024-11-27T13:40:44,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35019 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 8fc6390c5b3b9356bc4885792f770cf2 changed from -1.0 to 0.0, refreshing cache 2024-11-27T13:40:44,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35019 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region dcf91d39846df82f849110c861f2a7d0 changed from -1.0 to 0.0, refreshing cache 2024-11-27T13:40:44,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35019 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region a9a679b7eb3754995869ddbc90163b1c changed from -1.0 to 0.0, refreshing cache 2024-11-27T13:40:44,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35019 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 9c46624524e53a899138ac5a715a35d6 changed from -1.0 to 0.0, refreshing cache 2024-11-27T13:40:44,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35019 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 8700ed52628a2e71e47ec460999f2612 changed from -1.0 to 0.0, refreshing cache 2024-11-27T13:40:44,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35019 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 457728e6ba667138202e8a0dfa04b5b7 changed from -1.0 to 0.0, refreshing cache 2024-11-27T13:40:44,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35019 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 5c6bad09b1898b3a3810c7bfeb938da4 changed from -1.0 to 0.0, refreshing cache 2024-11-27T13:40:44,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35019 {}] rsgroup.RSGroupBasedLoadBalancer(138): Balancing RSGroup=default 2024-11-27T13:40:44,596 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35019 {}] rsgroup.RSGroupBasedLoadBalancer(151): Start Generate Balance plan for group: default 2024-11-27T13:40:44,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35019 {}] balancer.BaseLoadBalancer(619): Start Generate Balance plan for cluster. 2024-11-27T13:40:44,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35019 {}] balancer.BalancerClusterState(204): Hosts are {29b1b1ba230f=0} racks are {/default-rack=0} 2024-11-27T13:40:44,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35019 {}] balancer.BalancerClusterState(303): server 0 has 9 regions 2024-11-27T13:40:44,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35019 {}] balancer.BalancerClusterState(303): server 1 has 9 regions 2024-11-27T13:40:44,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35019 {}] balancer.BalancerClusterState(303): server 2 has 9 regions 2024-11-27T13:40:44,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35019 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-27T13:40:44,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35019 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-27T13:40:44,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35019 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-27T13:40:44,598 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35019 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-27T13:40:44,598 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35019 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-27T13:40:44,598 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35019 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-27T13:40:44,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35019 {}] balancer.BalancerClusterState(326): Number of tables=2, number of hosts=1, number of racks=1 2024-11-27T13:40:44,604 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35019 {}] balancer.StochasticLoadBalancer(395): Cluster wide - skipping load balancing because weighted average imbalance=0.016474464579901153 <= threshold(0.025). If you want more aggressive balancing, either lower hbase.master.balancer.stochastic.minCostNeedBalance from 0.025 or increase the relative multiplier(s) of the specific cost function(s). functionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.0); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.0); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); 2024-11-27T13:40:44,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35019 {}] master.HMaster(2203): Balancer is going into sleep until next period in 300000ms 2024-11-27T13:40:44,607 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [900,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T13:40:44,617 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35019 {}] master.MasterRpcServices(567): Client=jenkins//172.17.0.2 set balanceSwitch=false 2024-11-27T13:40:44,635 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.assignment.TestRollbackSCP#testFailAndRollback Thread=385, OpenFileDescriptor=643, MaxFileDescriptor=1048576, SystemLoadAverage=259, ProcessCount=11, AvailableMemoryMB=9418 2024-11-27T13:40:44,638 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table hbase:meta 2024-11-27T13:40:44,639 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=hbase:meta,,1 2024-11-27T13:40:44,640 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing 29b1b1ba230f,39411,1732714839711 2024-11-27T13:40:44,640 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 29b1b1ba230f,39411,1732714839711: Simulated kill ***** 2024-11-27T13:40:44,640 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-27T13:40:44,642 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-27T13:40:44,645 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-27T13:40:44,645 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-27T13:40:44,645 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-27T13:40:44,645 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 307183864 }, "NonHeapMemoryUsage": { "committed": 151322624, "init": 7667712, "max": -1, "used": 149051440 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "29b1b1ba230f" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "29b1b1ba230f" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.Context": "regionserver", "tag.Hostname": "29b1b1ba230f" } ] } 2024-11-27T13:40:44,648 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35019 {}] master.MasterRpcServices(700): 29b1b1ba230f,39411,1732714839711 reported a fatal error: ***** ABORTING region server 29b1b1ba230f,39411,1732714839711: Simulated kill ***** 2024-11-27T13:40:44,650 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '29b1b1ba230f,39411,1732714839711' ***** 2024-11-27T13:40:44,650 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-11-27T13:40:44,651 INFO [RS:1;29b1b1ba230f:39411 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-27T13:40:44,651 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [15,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T13:40:44,651 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-27T13:40:44,651 INFO [RS:1;29b1b1ba230f:39411 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-27T13:40:44,651 INFO [RS:1;29b1b1ba230f:39411 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-27T13:40:44,651 INFO [RS:1;29b1b1ba230f:39411 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-27T13:40:44,652 INFO [RS:1;29b1b1ba230f:39411 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;29b1b1ba230f:39411. 2024-11-27T13:40:44,652 DEBUG [RS:1;29b1b1ba230f:39411 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T13:40:44,659 DEBUG [RS:1;29b1b1ba230f:39411 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T13:40:44,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42991 is added to blk_1073741836_1012 (size=32114) 2024-11-27T13:40:44,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38955 is added to blk_1073741836_1012 (size=32114) 2024-11-27T13:40:44,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37007 is added to blk_1073741836_1012 (size=32114) 2024-11-27T13:40:44,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42991 is added to blk_1073741833_1009 (size=2237) 2024-11-27T13:40:44,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38955 is added to blk_1073741833_1009 (size=2237) 2024-11-27T13:40:44,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37007 is added to blk_1073741833_1009 (size=2237) 2024-11-27T13:40:44,673 DEBUG [RS:1;29b1b1ba230f:39411 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T13:40:44,673 INFO [RS:1;29b1b1ba230f:39411 {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T13:40:44,674 INFO [RS:1;29b1b1ba230f:39411 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39411 2024-11-27T13:40:44,718 INFO [regionserver/29b1b1ba230f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T13:40:44,749 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39411-0x1017c8633370002, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/29b1b1ba230f,39411,1732714839711 2024-11-27T13:40:44,749 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35019-0x1017c8633370000, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-27T13:40:44,749 INFO [RS:1;29b1b1ba230f:39411 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-27T13:40:44,763 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [29b1b1ba230f,39411,1732714839711] 2024-11-27T13:40:44,773 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/29b1b1ba230f,39411,1732714839711 already deleted, retry=false 2024-11-27T13:40:44,774 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 29b1b1ba230f,39411,1732714839711 on 29b1b1ba230f,35019,1732714838827 2024-11-27T13:40:44,781 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true 2024-11-27T13:40:44,786 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=57 for 29b1b1ba230f,39411,1732714839711 (carryingMeta=true) 29b1b1ba230f,39411,1732714839711/CRASHED/regionCount=9/lock=java.util.concurrent.locks.ReentrantReadWriteLock@17c80ef8[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-27T13:40:44,786 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true 2024-11-27T13:40:44,787 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true, isMeta: true 2024-11-27T13:40:44,789 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/WALs/29b1b1ba230f,39411,1732714839711-splitting 2024-11-27T13:40:44,791 INFO [PEWorker-1 {}] master.SplitWALManager(105): 29b1b1ba230f,39411,1732714839711 WAL count=1, meta=true 2024-11-27T13:40:44,793 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta}] 2024-11-27T13:40:44,797 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:44,800 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765}] 2024-11-27T13:40:44,853 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T13:40:44,863 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39411-0x1017c8633370002, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T13:40:44,864 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39411-0x1017c8633370002, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T13:40:44,864 INFO [RS:1;29b1b1ba230f:39411 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-27T13:40:44,864 INFO [RS:1;29b1b1ba230f:39411 {}] regionserver.HRegionServer(1031): Exiting; stopping=29b1b1ba230f,39411,1732714839711; zookeeper connection closed. 2024-11-27T13:40:44,865 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@67223e49 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@67223e49 2024-11-27T13:40:44,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44709 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=59 2024-11-27T13:40:44,996 INFO [RS_LOG_REPLAY_OPS-regionserver/29b1b1ba230f:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(299): Splitting hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/WALs/29b1b1ba230f,39411,1732714839711-splitting/29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, size=31.4 K (32114bytes) 2024-11-27T13:40:44,997 DEBUG [RS_LOG_REPLAY_OPS-regionserver/29b1b1ba230f:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-27T13:40:44,998 DEBUG [RS_LOG_REPLAY_OPS-regionserver/29b1b1ba230f:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-27T13:40:44,998 INFO [RS_LOG_REPLAY_OPS-regionserver/29b1b1ba230f:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/WALs/29b1b1ba230f,39411,1732714839711-splitting/29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta 2024-11-27T13:40:45,003 INFO [RS_LOG_REPLAY_OPS-regionserver/29b1b1ba230f:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/WALs/29b1b1ba230f,39411,1732714839711-splitting/29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta after 3ms 2024-11-27T13:40:45,012 INFO [RS_LOG_REPLAY_OPS-regionserver/29b1b1ba230f:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(310): Open hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/WALs/29b1b1ba230f,39411,1732714839711-splitting/29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta took 17ms 2024-11-27T13:40:45,023 DEBUG [RS_LOG_REPLAY_OPS-regionserver/29b1b1ba230f:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-27T13:40:45,030 INFO [RS_LOG_REPLAY_OPS-regionserver/29b1b1ba230f:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta.temp 2024-11-27T13:40:45,033 INFO [RS_LOG_REPLAY_OPS-regionserver/29b1b1ba230f:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta.temp 2024-11-27T13:40:45,035 DEBUG [RS_LOG_REPLAY_OPS-regionserver/29b1b1ba230f:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/WALs/29b1b1ba230f,39411,1732714839711-splitting/29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta so closing down 2024-11-27T13:40:45,036 DEBUG [RS_LOG_REPLAY_OPS-regionserver/29b1b1ba230f:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T13:40:45,038 INFO [RS_LOG_REPLAY_OPS-regionserver/29b1b1ba230f:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T13:40:45,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37007 is added to blk_1073741864_1040 (size=31873) 2024-11-27T13:40:45,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42991 is added to blk_1073741864_1040 (size=31873) 2024-11-27T13:40:45,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38955 is added to blk_1073741864_1040 (size=31873) 2024-11-27T13:40:45,050 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta.temp (wrote 57 edits, skipped 0 edits in 3 ms) 2024-11-27T13:40:45,052 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta.temp to hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-27T13:40:45,057 INFO [RS_LOG_REPLAY_OPS-regionserver/29b1b1ba230f:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(425): Processed 58 edits across 1 Regions in 38 ms; skipped=1; WAL=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/WALs/29b1b1ba230f,39411,1732714839711-splitting/29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, size=31.4 K, length=32114, corrupted=false, cancelled=false 2024-11-27T13:40:45,057 DEBUG [RS_LOG_REPLAY_OPS-regionserver/29b1b1ba230f:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(428): Completed split of hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/WALs/29b1b1ba230f,39411,1732714839711-splitting/29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, journal: Splitting hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/WALs/29b1b1ba230f,39411,1732714839711-splitting/29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, size=31.4 K (32114bytes) at 1732714844996Creating recovered edits writer path=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta.temp at 1732714845034 (+38 ms)Finishing writing output for hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/WALs/29b1b1ba230f,39411,1732714839711-splitting/29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta so closing down at 1732714845035 (+1 ms)3 split writer threads finished at 1732714845038 (+3 ms)Closed recovered edits writer path=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta.temp (wrote 57 edits, skipped 0 edits in 3 ms) at 1732714845050 (+12 ms)Rename recovered edits hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta.temp to hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 at 1732714845052 (+2 ms)Processed 58 edits across 1 Regions in 38 ms; skipped=1; WAL=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/WALs/29b1b1ba230f,39411,1732714839711-splitting/29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, size=31.4 K, length=32114, corrupted=false, cancelled=false at 1732714845057 (+5 ms) 2024-11-27T13:40:45,058 DEBUG [RS_LOG_REPLAY_OPS-regionserver/29b1b1ba230f:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/WALs/29b1b1ba230f,39411,1732714839711-splitting/29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta 2024-11-27T13:40:45,059 DEBUG [RS_LOG_REPLAY_OPS-regionserver/29b1b1ba230f:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-27T13:40:45,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35019 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-27T13:40:45,068 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/WALs/29b1b1ba230f,39411,1732714839711-splitting/29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta to hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/oldWALs 2024-11-27T13:40:45,070 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-27T13:40:45,070 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 in 268 msec 2024-11-27T13:40:45,072 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:45,075 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-27T13:40:45,075 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 in 280 msec 2024-11-27T13:40:45,078 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/WALs/29b1b1ba230f,39411,1732714839711-splitting dir is empty, no logs to split. 2024-11-27T13:40:45,078 INFO [PEWorker-1 {}] master.SplitWALManager(105): 29b1b1ba230f,39411,1732714839711 WAL count=0, meta=true 2024-11-27T13:40:45,078 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 29b1b1ba230f,39411,1732714839711 WAL splitting is done? wals=0, meta=true 2024-11-27T13:40:45,080 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-27T13:40:45,081 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T13:40:45,082 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-27T13:40:45,168 INFO [regionserver/29b1b1ba230f:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-27T13:40:45,169 INFO [regionserver/29b1b1ba230f:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-27T13:40:45,233 DEBUG [29b1b1ba230f:35019 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-27T13:40:45,233 DEBUG [29b1b1ba230f:35019 {}] balancer.BalancerClusterState(204): Hosts are {29b1b1ba230f=0} racks are {/default-rack=0} 2024-11-27T13:40:45,233 DEBUG [29b1b1ba230f:35019 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-27T13:40:45,233 DEBUG [29b1b1ba230f:35019 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-27T13:40:45,233 DEBUG [29b1b1ba230f:35019 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-27T13:40:45,233 DEBUG [29b1b1ba230f:35019 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-27T13:40:45,233 INFO [29b1b1ba230f:35019 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-27T13:40:45,233 INFO [29b1b1ba230f:35019 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-27T13:40:45,233 DEBUG [29b1b1ba230f:35019 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-27T13:40:45,234 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:45,235 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 29b1b1ba230f,38845,1732714839599, state=OPENING 2024-11-27T13:40:45,286 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35019-0x1017c8633370000, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:45,286 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44709-0x1017c8633370003, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:45,286 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1017c8633370001, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:45,286 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T13:40:45,286 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:45,286 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:45,286 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:45,286 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=29b1b1ba230f,38845,1732714839599}] 2024-11-27T13:40:45,446 INFO [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-27T13:40:45,446 INFO [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T13:40:45,447 INFO [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=61}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-27T13:40:45,450 INFO [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=29b1b1ba230f%2C38845%2C1732714839599.meta, suffix=.meta, logDir=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/WALs/29b1b1ba230f,38845,1732714839599, archiveDir=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/oldWALs, maxLogs=32 2024-11-27T13:40:45,467 DEBUG [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=61}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/WALs/29b1b1ba230f,38845,1732714839599/29b1b1ba230f%2C38845%2C1732714839599.meta.1732714845453.meta, exclude list is [], retry=0 2024-11-27T13:40:45,470 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42991,DS-d60122c6-962a-4258-9fbb-6d2f24e1c6ac,DISK] 2024-11-27T13:40:45,470 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38955,DS-fac1a99f-a8dd-4363-b07e-bd13cdba036b,DISK] 2024-11-27T13:40:45,470 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37007,DS-608d5eb3-8a33-4125-bd6d-2845739e66fc,DISK] 2024-11-27T13:40:45,473 INFO [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/WALs/29b1b1ba230f,38845,1732714839599/29b1b1ba230f%2C38845%2C1732714839599.meta.1732714845453.meta 2024-11-27T13:40:45,473 DEBUG [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34247:34247),(127.0.0.1/127.0.0.1:36243:36243),(127.0.0.1/127.0.0.1:36867:36867)] 2024-11-27T13:40:45,473 DEBUG [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-27T13:40:45,474 DEBUG [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=61}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-27T13:40:45,474 DEBUG [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-27T13:40:45,474 INFO [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-27T13:40:45,474 DEBUG [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-27T13:40:45,474 DEBUG [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T13:40:45,474 DEBUG [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-27T13:40:45,474 DEBUG [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-27T13:40:45,476 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-27T13:40:45,477 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-27T13:40:45,478 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T13:40:45,478 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T13:40:45,478 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-27T13:40:45,479 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-27T13:40:45,479 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T13:40:45,480 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T13:40:45,480 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-27T13:40:45,481 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-27T13:40:45,481 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T13:40:45,481 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T13:40:45,482 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-27T13:40:45,483 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-27T13:40:45,483 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T13:40:45,483 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T13:40:45,483 DEBUG [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-27T13:40:45,484 DEBUG [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/hbase/meta/1588230740 2024-11-27T13:40:45,487 DEBUG [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/hbase/meta/1588230740 2024-11-27T13:40:45,488 INFO [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-27T13:40:45,511 DEBUG [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5793): Applied 264, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=60, path=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-27T13:40:45,514 INFO [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=28 KB heapSize=48.31 KB 2024-11-27T13:40:45,576 DEBUG [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/hbase/meta/1588230740/.tmp/info/8b2bfb751a5d4abea77387204b88d8ea is 138, key is test,bbb,1732714842283.2b6b183b9bc56b2bc4dc6f8968ee01c8./info:regioninfo/1732714843349/Put/seqid=0 2024-11-27T13:40:45,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37007 is added to blk_1073741866_1042 (size=34924) 2024-11-27T13:40:45,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42991 is added to blk_1073741866_1042 (size=34924) 2024-11-27T13:40:45,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38955 is added to blk_1073741866_1042 (size=34924) 2024-11-27T13:40:45,589 INFO [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.86 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/hbase/meta/1588230740/.tmp/info/8b2bfb751a5d4abea77387204b88d8ea 2024-11-27T13:40:45,662 DEBUG [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/hbase/meta/1588230740/.tmp/ns/0d2089d75b964057b83e45560ea420f6 is 43, key is default/ns:d/1732714842019/Put/seqid=0 2024-11-27T13:40:45,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42991 is added to blk_1073741867_1043 (size=5153) 2024-11-27T13:40:45,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37007 is added to blk_1073741867_1043 (size=5153) 2024-11-27T13:40:45,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38955 is added to blk_1073741867_1043 (size=5153) 2024-11-27T13:40:45,670 INFO [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/hbase/meta/1588230740/.tmp/ns/0d2089d75b964057b83e45560ea420f6 2024-11-27T13:40:45,693 DEBUG [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/hbase/meta/1588230740/.tmp/table/b3b067169ca7433eb071d7e41bc1c85d is 40, key is test/table:state/1732714843697/Put/seqid=0 2024-11-27T13:40:45,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42991 is added to blk_1073741868_1044 (size=5165) 2024-11-27T13:40:45,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37007 is added to blk_1073741868_1044 (size=5165) 2024-11-27T13:40:45,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38955 is added to blk_1073741868_1044 (size=5165) 2024-11-27T13:40:45,703 INFO [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=72 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/hbase/meta/1588230740/.tmp/table/b3b067169ca7433eb071d7e41bc1c85d 2024-11-27T13:40:45,713 DEBUG [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/hbase/meta/1588230740/.tmp/info/8b2bfb751a5d4abea77387204b88d8ea as hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/hbase/meta/1588230740/info/8b2bfb751a5d4abea77387204b88d8ea 2024-11-27T13:40:45,723 INFO [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/hbase/meta/1588230740/info/8b2bfb751a5d4abea77387204b88d8ea, entries=260, sequenceid=60, filesize=34.1 K 2024-11-27T13:40:45,726 DEBUG [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/hbase/meta/1588230740/.tmp/ns/0d2089d75b964057b83e45560ea420f6 as hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/hbase/meta/1588230740/ns/0d2089d75b964057b83e45560ea420f6 2024-11-27T13:40:45,736 INFO [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/hbase/meta/1588230740/ns/0d2089d75b964057b83e45560ea420f6, entries=2, sequenceid=60, filesize=5.0 K 2024-11-27T13:40:45,738 DEBUG [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/hbase/meta/1588230740/.tmp/table/b3b067169ca7433eb071d7e41bc1c85d as hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/hbase/meta/1588230740/table/b3b067169ca7433eb071d7e41bc1c85d 2024-11-27T13:40:45,746 INFO [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/hbase/meta/1588230740/table/b3b067169ca7433eb071d7e41bc1c85d, entries=2, sequenceid=60, filesize=5.0 K 2024-11-27T13:40:45,751 INFO [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~28 KB/28672, heapSize ~48.02 KB/49168, currentSize=0 B/0 for 1588230740 in 236ms, sequenceid=60, compaction requested=false; wal=null 2024-11-27T13:40:45,753 DEBUG [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-27T13:40:45,756 DEBUG [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-27T13:40:45,758 DEBUG [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-27T13:40:45,758 DEBUG [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-27T13:40:45,759 DEBUG [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-27T13:40:45,761 DEBUG [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-27T13:40:45,764 DEBUG [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/hbase/meta/1588230740/recovered.edits/60.seqid, newMaxSeqId=60, maxSeqId=1 2024-11-27T13:40:45,766 INFO [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=61; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61550122, jitterRate=-0.08283171057701111}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-27T13:40:45,766 DEBUG [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-27T13:40:45,766 DEBUG [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732714845475Writing region info on filesystem at 1732714845475Initializing all the Stores at 1732714845476 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732714845476Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732714845476Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732714845476Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732714845476Obtaining lock to block concurrent updates at 1732714845514 (+38 ms)Preparing flush snapshotting stores in 1588230740 at 1732714845514Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=28672, getHeapSize=49408, getOffHeapSize=0, getCellsCount=264 at 1732714845517 (+3 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732714845517Flushing 1588230740/info: creating writer at 1732714845518 (+1 ms)Flushing 1588230740/info: appending metadata at 1732714845566 (+48 ms)Flushing 1588230740/info: closing flushed file at 1732714845569 (+3 ms)Flushing 1588230740/ns: creating writer at 1732714845646 (+77 ms)Flushing 1588230740/ns: appending metadata at 1732714845661 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1732714845661Flushing 1588230740/table: creating writer at 1732714845678 (+17 ms)Flushing 1588230740/table: appending metadata at 1732714845692 (+14 ms)Flushing 1588230740/table: closing flushed file at 1732714845692Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@29ef4364: reopening flushed file at 1732714845711 (+19 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1c40aae1: reopening flushed file at 1732714845725 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@14c1a245: reopening flushed file at 1732714845736 (+11 ms)Finished flush of dataSize ~28 KB/28672, heapSize ~48.02 KB/49168, currentSize=0 B/0 for 1588230740 in 236ms, sequenceid=60, compaction requested=false; wal=null at 1732714845752 (+16 ms)Cleaning up temporary data from old regions at 1732714845758 (+6 ms)Running coprocessor post-open hooks at 1732714845766 (+8 ms)Region opened successfully at 1732714845766 2024-11-27T13:40:45,767 INFO [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=61, masterSystemTime=1732714845440 2024-11-27T13:40:45,770 DEBUG [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-27T13:40:45,770 INFO [RS_OPEN_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-27T13:40:45,771 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=61, regionLocation=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:45,772 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 29b1b1ba230f,38845,1732714839599, state=OPEN 2024-11-27T13:40:45,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1017c8633370001, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:45,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44709-0x1017c8633370003, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:45,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35019-0x1017c8633370000, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:45,823 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:45,823 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:45,823 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:45,823 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=61, ppid=60, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:45,826 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-27T13:40:45,827 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=29b1b1ba230f,38845,1732714839599 in 537 msec 2024-11-27T13:40:45,830 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=57 2024-11-27T13:40:45,830 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 747 msec 2024-11-27T13:40:45,831 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 29b1b1ba230f,39411,1732714839711 had 9 regions 2024-11-27T13:40:45,832 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true, isMeta: false 2024-11-27T13:40:45,836 INFO [PEWorker-1 {}] master.SplitWALManager(105): 29b1b1ba230f,39411,1732714839711 WAL count=1, meta=false 2024-11-27T13:40:45,836 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272}] 2024-11-27T13:40:45,839 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:45,840 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765}] 2024-11-27T13:40:45,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44709 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=63 2024-11-27T13:40:46,010 INFO [RS_LOG_REPLAY_OPS-regionserver/29b1b1ba230f:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(299): Splitting hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/WALs/29b1b1ba230f,39411,1732714839711-splitting/29b1b1ba230f%2C39411%2C1732714839711.1732714841272, size=2.2 K (2237bytes) 2024-11-27T13:40:46,010 INFO [RS_LOG_REPLAY_OPS-regionserver/29b1b1ba230f:0-1 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/WALs/29b1b1ba230f,39411,1732714839711-splitting/29b1b1ba230f%2C39411%2C1732714839711.1732714841272 2024-11-27T13:40:46,011 INFO [RS_LOG_REPLAY_OPS-regionserver/29b1b1ba230f:0-1 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/WALs/29b1b1ba230f,39411,1732714839711-splitting/29b1b1ba230f%2C39411%2C1732714839711.1732714841272 after 1ms 2024-11-27T13:40:46,014 INFO [RS_LOG_REPLAY_OPS-regionserver/29b1b1ba230f:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(310): Open hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/WALs/29b1b1ba230f,39411,1732714839711-splitting/29b1b1ba230f%2C39411%2C1732714839711.1732714841272 took 4ms 2024-11-27T13:40:46,017 DEBUG [RS_LOG_REPLAY_OPS-regionserver/29b1b1ba230f:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for e7a7c9c02e40375cd974ffe51b6da75b: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-27T13:40:46,019 DEBUG [RS_LOG_REPLAY_OPS-regionserver/29b1b1ba230f:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 2b6b183b9bc56b2bc4dc6f8968ee01c8: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-27T13:40:46,021 DEBUG [RS_LOG_REPLAY_OPS-regionserver/29b1b1ba230f:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for f5f0e25bfe9d22339132aab7b2f16f09: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-27T13:40:46,023 DEBUG [RS_LOG_REPLAY_OPS-regionserver/29b1b1ba230f:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 9ee34ecbedcb7dfdabcb99a67ac21045: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-27T13:40:46,025 DEBUG [RS_LOG_REPLAY_OPS-regionserver/29b1b1ba230f:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for c4f747499d4a9438c3cddbfb1ead0d2b: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-27T13:40:46,027 DEBUG [RS_LOG_REPLAY_OPS-regionserver/29b1b1ba230f:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for de4deb0c16ab89c9c56fc8912227f1c5: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-27T13:40:46,029 DEBUG [RS_LOG_REPLAY_OPS-regionserver/29b1b1ba230f:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for d64103cc558d5243fdfd1e469558a599: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-27T13:40:46,031 DEBUG [RS_LOG_REPLAY_OPS-regionserver/29b1b1ba230f:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for cccf7ce6a567350b0ae162454b0a9379: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-27T13:40:46,032 DEBUG [RS_LOG_REPLAY_OPS-regionserver/29b1b1ba230f:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/WALs/29b1b1ba230f,39411,1732714839711-splitting/29b1b1ba230f%2C39411%2C1732714839711.1732714841272 so closing down 2024-11-27T13:40:46,032 DEBUG [RS_LOG_REPLAY_OPS-regionserver/29b1b1ba230f:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T13:40:46,032 INFO [RS_LOG_REPLAY_OPS-regionserver/29b1b1ba230f:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T13:40:46,032 INFO [RS_LOG_REPLAY_OPS-regionserver/29b1b1ba230f:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(425): Processed 8 edits across 0 Regions in 18 ms; skipped=8; WAL=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/WALs/29b1b1ba230f,39411,1732714839711-splitting/29b1b1ba230f%2C39411%2C1732714839711.1732714841272, size=2.2 K, length=2237, corrupted=false, cancelled=false 2024-11-27T13:40:46,032 DEBUG [RS_LOG_REPLAY_OPS-regionserver/29b1b1ba230f:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(428): Completed split of hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/WALs/29b1b1ba230f,39411,1732714839711-splitting/29b1b1ba230f%2C39411%2C1732714839711.1732714841272, journal: Splitting hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/WALs/29b1b1ba230f,39411,1732714839711-splitting/29b1b1ba230f%2C39411%2C1732714839711.1732714841272, size=2.2 K (2237bytes) at 1732714846010Finishing writing output for hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/WALs/29b1b1ba230f,39411,1732714839711-splitting/29b1b1ba230f%2C39411%2C1732714839711.1732714841272 so closing down at 1732714846032 (+22 ms)3 split writer threads finished at 1732714846032Processed 8 edits across 0 Regions in 18 ms; skipped=8; WAL=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/WALs/29b1b1ba230f,39411,1732714839711-splitting/29b1b1ba230f%2C39411%2C1732714839711.1732714841272, size=2.2 K, length=2237, corrupted=false, cancelled=false at 1732714846032 2024-11-27T13:40:46,032 DEBUG [RS_LOG_REPLAY_OPS-regionserver/29b1b1ba230f:0-1 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/WALs/29b1b1ba230f,39411,1732714839711-splitting/29b1b1ba230f%2C39411%2C1732714839711.1732714841272 2024-11-27T13:40:46,032 DEBUG [RS_LOG_REPLAY_OPS-regionserver/29b1b1ba230f:0-1 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-27T13:40:46,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35019 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-27T13:40:46,038 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/WALs/29b1b1ba230f,39411,1732714839711-splitting/29b1b1ba230f%2C39411%2C1732714839711.1732714841272 to hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/oldWALs 2024-11-27T13:40:46,042 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-27T13:40:46,042 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 in 198 msec 2024-11-27T13:40:46,044 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,046 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=57 2024-11-27T13:40:46,046 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 in 208 msec 2024-11-27T13:40:46,049 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/WALs/29b1b1ba230f,39411,1732714839711-splitting dir is empty, no logs to split. 2024-11-27T13:40:46,049 INFO [PEWorker-1 {}] master.SplitWALManager(105): 29b1b1ba230f,39411,1732714839711 WAL count=0, meta=false 2024-11-27T13:40:46,049 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 29b1b1ba230f,39411,1732714839711 WAL splitting is done? wals=0, meta=false 2024-11-27T13:40:46,051 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN}, {pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN}, {pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN}, {pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f5f0e25bfe9d22339132aab7b2f16f09, ASSIGN}, {pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e7a7c9c02e40375cd974ffe51b6da75b, ASSIGN}, {pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=de4deb0c16ab89c9c56fc8912227f1c5, ASSIGN}, {pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d64103cc558d5243fdfd1e469558a599, ASSIGN}, {pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=cccf7ce6a567350b0ae162454b0a9379, ASSIGN}] 2024-11-27T13:40:46,054 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=cccf7ce6a567350b0ae162454b0a9379, ASSIGN 2024-11-27T13:40:46,055 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=cccf7ce6a567350b0ae162454b0a9379, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-27T13:40:46,057 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d64103cc558d5243fdfd1e469558a599, ASSIGN 2024-11-27T13:40:46,058 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d64103cc558d5243fdfd1e469558a599, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-27T13:40:46,059 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=de4deb0c16ab89c9c56fc8912227f1c5, ASSIGN 2024-11-27T13:40:46,060 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=de4deb0c16ab89c9c56fc8912227f1c5, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-27T13:40:46,061 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e7a7c9c02e40375cd974ffe51b6da75b, ASSIGN 2024-11-27T13:40:46,062 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=e7a7c9c02e40375cd974ffe51b6da75b, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-27T13:40:46,063 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f5f0e25bfe9d22339132aab7b2f16f09, ASSIGN 2024-11-27T13:40:46,064 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f5f0e25bfe9d22339132aab7b2f16f09, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-27T13:40:46,065 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN 2024-11-27T13:40:46,066 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-27T13:40:46,067 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN 2024-11-27T13:40:46,068 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-27T13:40:46,068 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN 2024-11-27T13:40:46,069 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-27T13:40:46,207 DEBUG [29b1b1ba230f:35019 {}] balancer.BalancerClusterState(204): Hosts are {29b1b1ba230f=0} racks are {/default-rack=0} 2024-11-27T13:40:46,207 DEBUG [29b1b1ba230f:35019 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-27T13:40:46,208 DEBUG [29b1b1ba230f:35019 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-27T13:40:46,208 DEBUG [29b1b1ba230f:35019 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-27T13:40:46,208 DEBUG [29b1b1ba230f:35019 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-27T13:40:46,208 INFO [29b1b1ba230f:35019 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-27T13:40:46,208 INFO [29b1b1ba230f:35019 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-27T13:40:46,208 DEBUG [29b1b1ba230f:35019 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-27T13:40:46,209 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=d64103cc558d5243fdfd1e469558a599, regionState=OPENING, regionLocation=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,212 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=f5f0e25bfe9d22339132aab7b2f16f09, regionState=OPENING, regionLocation=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,213 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=cccf7ce6a567350b0ae162454b0a9379, regionState=OPENING, regionLocation=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,212 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 29b1b1ba230f:39411 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 29b1b1ba230f/172.17.0.2:39411 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T13:40:46,214 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=65 updating hbase:meta row=9ee34ecbedcb7dfdabcb99a67ac21045, regionState=OPENING, regionLocation=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,215 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=c4f747499d4a9438c3cddbfb1ead0d2b, regionState=OPENING, regionLocation=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:46,216 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=de4deb0c16ab89c9c56fc8912227f1c5, regionState=OPENING, regionLocation=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:46,216 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=29b1b1ba230f,39411,1732714839711, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=29b1b1ba230f,39411,1732714839711, seqNum=-1, error=java.net.ConnectException: Call to address=29b1b1ba230f:39411 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 29b1b1ba230f/172.17.0.2:39411 2024-11-27T13:40:46,217 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=e7a7c9c02e40375cd974ffe51b6da75b, regionState=OPENING, regionLocation=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:46,218 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=29b1b1ba230f,39411,1732714839711, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-27T13:40:46,218 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=29b1b1ba230f,39411,1732714839711, seqNum=-1 from cache 2024-11-27T13:40:46,218 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=2b6b183b9bc56b2bc4dc6f8968ee01c8, regionState=OPENING, regionLocation=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:46,218 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T13:40:46,218 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=29b1b1ba230f,38845,1732714839599, seqNum=-1] 2024-11-27T13:40:46,219 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T13:40:46,220 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 29b1b1ba230f:39411 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 29b1b1ba230f/172.17.0.2:39411 2024-11-27T13:40:46,221 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 29b1b1ba230f:39411 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 29b1b1ba230f/172.17.0.2:39411 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T13:40:46,221 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=29b1b1ba230f,39411,1732714839711, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=29b1b1ba230f,38845,1732714839599, seqNum=-1, error=java.net.ConnectException: Call to address=29b1b1ba230f:39411 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 29b1b1ba230f/172.17.0.2:39411 2024-11-27T13:40:46,221 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=29b1b1ba230f,39411,1732714839711, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=29b1b1ba230f,38845,1732714839599, seqNum=-1, error=java.net.ConnectException: Call to address=29b1b1ba230f:39411 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 29b1b1ba230f/172.17.0.2:39411 2024-11-27T13:40:46,222 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47017, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T13:40:46,222 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=29b1b1ba230f,39411,1732714839711, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=29b1b1ba230f,38845,1732714839599, seqNum=-1, error=java.net.ConnectException: Call to address=29b1b1ba230f:39411 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 29b1b1ba230f/172.17.0.2:39411 2024-11-27T13:40:46,222 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=29b1b1ba230f,39411,1732714839711, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=29b1b1ba230f,38845,1732714839599, seqNum=-1, error=java.net.ConnectException: Call to address=29b1b1ba230f:39411 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 29b1b1ba230f/172.17.0.2:39411 2024-11-27T13:40:46,222 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=29b1b1ba230f,39411,1732714839711, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=29b1b1ba230f,38845,1732714839599, seqNum=-1, error=java.net.ConnectException: Call to address=29b1b1ba230f:39411 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 29b1b1ba230f/172.17.0.2:39411 2024-11-27T13:40:46,222 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=29b1b1ba230f,39411,1732714839711, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=29b1b1ba230f,38845,1732714839599, seqNum=-1, error=java.net.ConnectException: Call to address=29b1b1ba230f:39411 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 29b1b1ba230f/172.17.0.2:39411 2024-11-27T13:40:46,222 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 29b1b1ba230f:39411 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 29b1b1ba230f/172.17.0.2:39411 2024-11-27T13:40:46,224 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN because future has completed 2024-11-27T13:40:46,224 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b6b183b9bc56b2bc4dc6f8968ee01c8, server=29b1b1ba230f,38845,1732714839599}] 2024-11-27T13:40:46,323 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d64103cc558d5243fdfd1e469558a599, ASSIGN because future has completed 2024-11-27T13:40:46,324 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure d64103cc558d5243fdfd1e469558a599, server=29b1b1ba230f,44709,1732714839765}] 2024-11-27T13:40:46,334 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f5f0e25bfe9d22339132aab7b2f16f09, ASSIGN because future has completed 2024-11-27T13:40:46,335 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5f0e25bfe9d22339132aab7b2f16f09, server=29b1b1ba230f,44709,1732714839765}] 2024-11-27T13:40:46,335 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=cccf7ce6a567350b0ae162454b0a9379, ASSIGN because future has completed 2024-11-27T13:40:46,336 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN because future has completed 2024-11-27T13:40:46,338 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN because future has completed 2024-11-27T13:40:46,340 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=de4deb0c16ab89c9c56fc8912227f1c5, ASSIGN because future has completed 2024-11-27T13:40:46,340 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599}] 2024-11-27T13:40:46,341 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=e7a7c9c02e40375cd974ffe51b6da75b, ASSIGN because future has completed 2024-11-27T13:40:46,346 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure e7a7c9c02e40375cd974ffe51b6da75b, server=29b1b1ba230f,38845,1732714839599}] 2024-11-27T13:40:46,350 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure c4f747499d4a9438c3cddbfb1ead0d2b, server=29b1b1ba230f,38845,1732714839599}] 2024-11-27T13:40:46,354 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9ee34ecbedcb7dfdabcb99a67ac21045, server=29b1b1ba230f,44709,1732714839765}] 2024-11-27T13:40:46,357 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure cccf7ce6a567350b0ae162454b0a9379, server=29b1b1ba230f,44709,1732714839765}] 2024-11-27T13:40:46,380 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(132): Open test,ppp,1732714842283.de4deb0c16ab89c9c56fc8912227f1c5. 2024-11-27T13:40:46,380 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7752): Opening region: {ENCODED => de4deb0c16ab89c9c56fc8912227f1c5, NAME => 'test,ppp,1732714842283.de4deb0c16ab89c9c56fc8912227f1c5.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-27T13:40:46,381 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test de4deb0c16ab89c9c56fc8912227f1c5 2024-11-27T13:40:46,381 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(898): Instantiated test,ppp,1732714842283.de4deb0c16ab89c9c56fc8912227f1c5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T13:40:46,381 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7794): checking encryption for de4deb0c16ab89c9c56fc8912227f1c5 2024-11-27T13:40:46,381 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7797): checking classloading for de4deb0c16ab89c9c56fc8912227f1c5 2024-11-27T13:40:46,382 INFO [StoreOpener-de4deb0c16ab89c9c56fc8912227f1c5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region de4deb0c16ab89c9c56fc8912227f1c5 2024-11-27T13:40:46,383 INFO [StoreOpener-de4deb0c16ab89c9c56fc8912227f1c5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de4deb0c16ab89c9c56fc8912227f1c5 columnFamilyName family 2024-11-27T13:40:46,383 DEBUG [StoreOpener-de4deb0c16ab89c9c56fc8912227f1c5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T13:40:46,384 INFO [StoreOpener-de4deb0c16ab89c9c56fc8912227f1c5-1 {}] regionserver.HStore(327): Store=de4deb0c16ab89c9c56fc8912227f1c5/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T13:40:46,384 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1038): replaying wal for de4deb0c16ab89c9c56fc8912227f1c5 2024-11-27T13:40:46,385 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/de4deb0c16ab89c9c56fc8912227f1c5 2024-11-27T13:40:46,386 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/de4deb0c16ab89c9c56fc8912227f1c5 2024-11-27T13:40:46,387 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1048): stopping wal replay for de4deb0c16ab89c9c56fc8912227f1c5 2024-11-27T13:40:46,387 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1060): Cleaning up temporary data for de4deb0c16ab89c9c56fc8912227f1c5 2024-11-27T13:40:46,389 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): writing seq id for de4deb0c16ab89c9c56fc8912227f1c5 2024-11-27T13:40:46,390 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1114): Opened de4deb0c16ab89c9c56fc8912227f1c5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72365565, jitterRate=0.07833094894886017}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T13:40:46,390 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1122): Running coprocessor post-open hooks for de4deb0c16ab89c9c56fc8912227f1c5 2024-11-27T13:40:46,390 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1006): Region open journal for de4deb0c16ab89c9c56fc8912227f1c5: Running coprocessor pre-open hook at 1732714846381Writing region info on filesystem at 1732714846381Initializing all the Stores at 1732714846382 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732714846382Cleaning up temporary data from old regions at 1732714846387 (+5 ms)Running coprocessor post-open hooks at 1732714846390 (+3 ms)Region opened successfully at 1732714846390 2024-11-27T13:40:46,391 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1732714842283.de4deb0c16ab89c9c56fc8912227f1c5., pid=75, masterSystemTime=1732714846376 2024-11-27T13:40:46,393 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ppp,1732714842283.de4deb0c16ab89c9c56fc8912227f1c5. 2024-11-27T13:40:46,394 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(153): Opened test,ppp,1732714842283.de4deb0c16ab89c9c56fc8912227f1c5. 2024-11-27T13:40:46,394 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(132): Open test,vvv,1732714842283.c4f747499d4a9438c3cddbfb1ead0d2b. 2024-11-27T13:40:46,394 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7752): Opening region: {ENCODED => c4f747499d4a9438c3cddbfb1ead0d2b, NAME => 'test,vvv,1732714842283.c4f747499d4a9438c3cddbfb1ead0d2b.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-27T13:40:46,394 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c4f747499d4a9438c3cddbfb1ead0d2b 2024-11-27T13:40:46,394 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(898): Instantiated test,vvv,1732714842283.c4f747499d4a9438c3cddbfb1ead0d2b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T13:40:46,394 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7794): checking encryption for c4f747499d4a9438c3cddbfb1ead0d2b 2024-11-27T13:40:46,394 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7797): checking classloading for c4f747499d4a9438c3cddbfb1ead0d2b 2024-11-27T13:40:46,395 WARN [PEWorker-1 {}] procedure2.ProcedureTestingUtility(201): Set Kill before store update in rollback to: true 2024-11-27T13:40:46,396 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:46,396 INFO [StoreOpener-c4f747499d4a9438c3cddbfb1ead0d2b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c4f747499d4a9438c3cddbfb1ead0d2b 2024-11-27T13:40:46,397 INFO [StoreOpener-c4f747499d4a9438c3cddbfb1ead0d2b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c4f747499d4a9438c3cddbfb1ead0d2b columnFamilyName family 2024-11-27T13:40:46,397 DEBUG [StoreOpener-c4f747499d4a9438c3cddbfb1ead0d2b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T13:40:46,396 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1844): CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599 java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-27T13:40:46,397 INFO [StoreOpener-c4f747499d4a9438c3cddbfb1ead0d2b-1 {}] regionserver.HStore(327): Store=c4f747499d4a9438c3cddbfb1ead0d2b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T13:40:46,397 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1038): replaying wal for c4f747499d4a9438c3cddbfb1ead0d2b 2024-11-27T13:40:46,398 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/c4f747499d4a9438c3cddbfb1ead0d2b 2024-11-27T13:40:46,399 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/c4f747499d4a9438c3cddbfb1ead0d2b 2024-11-27T13:40:46,400 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1048): stopping wal replay for c4f747499d4a9438c3cddbfb1ead0d2b 2024-11-27T13:40:46,400 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1060): Cleaning up temporary data for c4f747499d4a9438c3cddbfb1ead0d2b 2024-11-27T13:40:46,402 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1093): writing seq id for c4f747499d4a9438c3cddbfb1ead0d2b 2024-11-27T13:40:46,403 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1114): Opened c4f747499d4a9438c3cddbfb1ead0d2b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67785391, jitterRate=0.010081037878990173}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T13:40:46,403 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c4f747499d4a9438c3cddbfb1ead0d2b 2024-11-27T13:40:46,403 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1006): Region open journal for c4f747499d4a9438c3cddbfb1ead0d2b: Running coprocessor pre-open hook at 1732714846394Writing region info on filesystem at 1732714846394Initializing all the Stores at 1732714846395 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732714846395Cleaning up temporary data from old regions at 1732714846400 (+5 ms)Running coprocessor post-open hooks at 1732714846403 (+3 ms)Region opened successfully at 1732714846403 2024-11-27T13:40:46,404 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1732714842283.c4f747499d4a9438c3cddbfb1ead0d2b., pid=77, masterSystemTime=1732714846376 2024-11-27T13:40:46,406 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1732714842283.c4f747499d4a9438c3cddbfb1ead0d2b. 2024-11-27T13:40:46,406 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(153): Opened test,vvv,1732714842283.c4f747499d4a9438c3cddbfb1ead0d2b. 2024-11-27T13:40:46,406 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(132): Open test,jjj,1732714842283.e7a7c9c02e40375cd974ffe51b6da75b. 2024-11-27T13:40:46,406 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7752): Opening region: {ENCODED => e7a7c9c02e40375cd974ffe51b6da75b, NAME => 'test,jjj,1732714842283.e7a7c9c02e40375cd974ffe51b6da75b.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-27T13:40:46,406 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test e7a7c9c02e40375cd974ffe51b6da75b 2024-11-27T13:40:46,406 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(898): Instantiated test,jjj,1732714842283.e7a7c9c02e40375cd974ffe51b6da75b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T13:40:46,407 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7794): checking encryption for e7a7c9c02e40375cd974ffe51b6da75b 2024-11-27T13:40:46,407 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7797): checking classloading for e7a7c9c02e40375cd974ffe51b6da75b 2024-11-27T13:40:46,408 INFO [StoreOpener-e7a7c9c02e40375cd974ffe51b6da75b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region e7a7c9c02e40375cd974ffe51b6da75b 2024-11-27T13:40:46,406 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=true, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1845) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 4 more 2024-11-27T13:40:46,409 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-27T13:40:46,409 INFO [StoreOpener-e7a7c9c02e40375cd974ffe51b6da75b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7a7c9c02e40375cd974ffe51b6da75b columnFamilyName family 2024-11-27T13:40:46,409 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=79, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure cccf7ce6a567350b0ae162454b0a9379, server=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,409 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T13:40:46,409 DEBUG [StoreOpener-e7a7c9c02e40375cd974ffe51b6da75b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T13:40:46,410 INFO [StoreOpener-e7a7c9c02e40375cd974ffe51b6da75b-1 {}] regionserver.HStore(327): Store=e7a7c9c02e40375cd974ffe51b6da75b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T13:40:46,410 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1038): replaying wal for e7a7c9c02e40375cd974ffe51b6da75b 2024-11-27T13:40:46,410 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=79, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure cccf7ce6a567350b0ae162454b0a9379, server=29b1b1ba230f,44709,1732714839765 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-27T13:40:46,411 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/e7a7c9c02e40375cd974ffe51b6da75b 2024-11-27T13:40:46,413 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/e7a7c9c02e40375cd974ffe51b6da75b 2024-11-27T13:40:46,413 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1048): stopping wal replay for e7a7c9c02e40375cd974ffe51b6da75b 2024-11-27T13:40:46,413 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1060): Cleaning up temporary data for e7a7c9c02e40375cd974ffe51b6da75b 2024-11-27T13:40:46,416 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1093): writing seq id for e7a7c9c02e40375cd974ffe51b6da75b 2024-11-27T13:40:46,417 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1114): Opened e7a7c9c02e40375cd974ffe51b6da75b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63993946, jitterRate=-0.046415895223617554}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T13:40:46,418 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e7a7c9c02e40375cd974ffe51b6da75b 2024-11-27T13:40:46,418 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1006): Region open journal for e7a7c9c02e40375cd974ffe51b6da75b: Running coprocessor pre-open hook at 1732714846407Writing region info on filesystem at 1732714846407Initializing all the Stores at 1732714846408 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732714846408Cleaning up temporary data from old regions at 1732714846413 (+5 ms)Running coprocessor post-open hooks at 1732714846418 (+5 ms)Region opened successfully at 1732714846418 2024-11-27T13:40:46,419 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2236): Post open deploy tasks for test,jjj,1732714842283.e7a7c9c02e40375cd974ffe51b6da75b., pid=76, masterSystemTime=1732714846376 2024-11-27T13:40:46,421 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2266): Finished post open deploy task for test,jjj,1732714842283.e7a7c9c02e40375cd974ffe51b6da75b. 2024-11-27T13:40:46,421 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(153): Opened test,jjj,1732714842283.e7a7c9c02e40375cd974ffe51b6da75b. 2024-11-27T13:40:46,421 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] handler.AssignRegionHandler(132): Open test,bbb,1732714842283.2b6b183b9bc56b2bc4dc6f8968ee01c8. 2024-11-27T13:40:46,421 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7752): Opening region: {ENCODED => 2b6b183b9bc56b2bc4dc6f8968ee01c8, NAME => 'test,bbb,1732714842283.2b6b183b9bc56b2bc4dc6f8968ee01c8.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-11-27T13:40:46,422 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 2b6b183b9bc56b2bc4dc6f8968ee01c8 2024-11-27T13:40:46,422 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(898): Instantiated test,bbb,1732714842283.2b6b183b9bc56b2bc4dc6f8968ee01c8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T13:40:46,422 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7794): checking encryption for 2b6b183b9bc56b2bc4dc6f8968ee01c8 2024-11-27T13:40:46,422 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7797): checking classloading for 2b6b183b9bc56b2bc4dc6f8968ee01c8 2024-11-27T13:40:46,423 INFO [StoreOpener-2b6b183b9bc56b2bc4dc6f8968ee01c8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 2b6b183b9bc56b2bc4dc6f8968ee01c8 2024-11-27T13:40:46,424 INFO [StoreOpener-2b6b183b9bc56b2bc4dc6f8968ee01c8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2b6b183b9bc56b2bc4dc6f8968ee01c8 columnFamilyName family 2024-11-27T13:40:46,424 DEBUG [StoreOpener-2b6b183b9bc56b2bc4dc6f8968ee01c8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T13:40:46,425 INFO [StoreOpener-2b6b183b9bc56b2bc4dc6f8968ee01c8-1 {}] regionserver.HStore(327): Store=2b6b183b9bc56b2bc4dc6f8968ee01c8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T13:40:46,425 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1038): replaying wal for 2b6b183b9bc56b2bc4dc6f8968ee01c8 2024-11-27T13:40:46,426 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/2b6b183b9bc56b2bc4dc6f8968ee01c8 2024-11-27T13:40:46,427 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/2b6b183b9bc56b2bc4dc6f8968ee01c8 2024-11-27T13:40:46,428 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1048): stopping wal replay for 2b6b183b9bc56b2bc4dc6f8968ee01c8 2024-11-27T13:40:46,428 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1060): Cleaning up temporary data for 2b6b183b9bc56b2bc4dc6f8968ee01c8 2024-11-27T13:40:46,430 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1093): writing seq id for 2b6b183b9bc56b2bc4dc6f8968ee01c8 2024-11-27T13:40:46,431 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1114): Opened 2b6b183b9bc56b2bc4dc6f8968ee01c8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71683656, jitterRate=0.06816971302032471}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T13:40:46,431 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2b6b183b9bc56b2bc4dc6f8968ee01c8 2024-11-27T13:40:46,431 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1006): Region open journal for 2b6b183b9bc56b2bc4dc6f8968ee01c8: Running coprocessor pre-open hook at 1732714846422Writing region info on filesystem at 1732714846422Initializing all the Stores at 1732714846423 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732714846423Cleaning up temporary data from old regions at 1732714846428 (+5 ms)Running coprocessor post-open hooks at 1732714846431 (+3 ms)Region opened successfully at 1732714846431 2024-11-27T13:40:46,432 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionServer(2236): Post open deploy tasks for test,bbb,1732714842283.2b6b183b9bc56b2bc4dc6f8968ee01c8., pid=72, masterSystemTime=1732714846376 2024-11-27T13:40:46,434 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionServer(2266): Finished post open deploy task for test,bbb,1732714842283.2b6b183b9bc56b2bc4dc6f8968ee01c8. 2024-11-27T13:40:46,435 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] handler.AssignRegionHandler(153): Opened test,bbb,1732714842283.2b6b183b9bc56b2bc4dc6f8968ee01c8. 2024-11-27T13:40:46,456 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-27T13:40:46,456 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T13:40:46,457 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-27T13:40:46,457 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T13:40:46,458 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-27T13:40:46,458 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-27T13:40:46,458 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-27T13:40:46,458 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T13:40:46,459 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-27T13:40:46,467 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-27T13:40:46,468 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,468 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T13:40:46,468 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,468 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T13:40:46,468 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,468 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T13:40:46,468 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,468 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T13:40:46,468 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,468 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T13:40:46,468 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,468 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T13:40:46,469 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,469 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T13:40:46,469 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,469 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T13:40:46,469 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T13:40:46,469 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T13:40:46,469 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,469 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T13:40:46,469 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T13:40:46,469 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T13:40:46,470 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T13:40:46,472 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-27T13:40:46,478 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true 2024-11-27T13:40:46,479 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,479 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,479 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T13:40:46,479 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:46,479 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,479 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,479 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN 2024-11-27T13:40:46,479 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN 2024-11-27T13:40:46,479 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN 2024-11-27T13:40:46,479 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f0e25bfe9d22339132aab7b2f16f09, ASSIGN 2024-11-27T13:40:46,479 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e7a7c9c02e40375cd974ffe51b6da75b, ASSIGN 2024-11-27T13:40:46,479 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de4deb0c16ab89c9c56fc8912227f1c5, ASSIGN 2024-11-27T13:40:46,479 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d64103cc558d5243fdfd1e469558a599, ASSIGN 2024-11-27T13:40:46,479 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cccf7ce6a567350b0ae162454b0a9379, ASSIGN 2024-11-27T13:40:46,480 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b6b183b9bc56b2bc4dc6f8968ee01c8, server=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:46,480 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure d64103cc558d5243fdfd1e469558a599, server=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,480 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5f0e25bfe9d22339132aab7b2f16f09, server=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,480 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:46,480 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure e7a7c9c02e40375cd974ffe51b6da75b, server=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:46,480 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure c4f747499d4a9438c3cddbfb1ead0d2b, server=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:46,480 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9ee34ecbedcb7dfdabcb99a67ac21045, server=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,480 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure cccf7ce6a567350b0ae162454b0a9379, server=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,480 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,480 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,480 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,480 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,480 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=29b1b1ba230f,38845,1732714839599 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,480 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,481 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,481 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,481 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN 2024-11-27T13:40:46,481 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(132): Open test,yyy,1732714842283.d64103cc558d5243fdfd1e469558a599. 2024-11-27T13:40:46,481 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,481 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN 2024-11-27T13:40:46,481 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,481 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7752): Opening region: {ENCODED => d64103cc558d5243fdfd1e469558a599, NAME => 'test,yyy,1732714842283.d64103cc558d5243fdfd1e469558a599.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-27T13:40:46,481 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN 2024-11-27T13:40:46,481 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f0e25bfe9d22339132aab7b2f16f09, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,481 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f0e25bfe9d22339132aab7b2f16f09, ASSIGN 2024-11-27T13:40:46,481 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e7a7c9c02e40375cd974ffe51b6da75b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,481 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e7a7c9c02e40375cd974ffe51b6da75b, ASSIGN 2024-11-27T13:40:46,481 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de4deb0c16ab89c9c56fc8912227f1c5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,481 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d64103cc558d5243fdfd1e469558a599 2024-11-27T13:40:46,481 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de4deb0c16ab89c9c56fc8912227f1c5, ASSIGN 2024-11-27T13:40:46,482 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d64103cc558d5243fdfd1e469558a599, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,482 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(898): Instantiated test,yyy,1732714842283.d64103cc558d5243fdfd1e469558a599.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T13:40:46,482 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d64103cc558d5243fdfd1e469558a599, ASSIGN 2024-11-27T13:40:46,482 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7794): checking encryption for d64103cc558d5243fdfd1e469558a599 2024-11-27T13:40:46,482 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cccf7ce6a567350b0ae162454b0a9379, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,482 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7797): checking classloading for d64103cc558d5243fdfd1e469558a599 2024-11-27T13:40:46,482 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cccf7ce6a567350b0ae162454b0a9379, ASSIGN 2024-11-27T13:40:46,482 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b6b183b9bc56b2bc4dc6f8968ee01c8, server=29b1b1ba230f,38845,1732714839599 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,482 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure d64103cc558d5243fdfd1e469558a599, server=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,482 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5f0e25bfe9d22339132aab7b2f16f09, server=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,482 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,482 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure e7a7c9c02e40375cd974ffe51b6da75b, server=29b1b1ba230f,38845,1732714839599 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,482 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure c4f747499d4a9438c3cddbfb1ead0d2b, server=29b1b1ba230f,38845,1732714839599 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,482 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9ee34ecbedcb7dfdabcb99a67ac21045, server=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,482 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure cccf7ce6a567350b0ae162454b0a9379, server=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,483 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 24 msec 2024-11-27T13:40:46,484 INFO [StoreOpener-d64103cc558d5243fdfd1e469558a599-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d64103cc558d5243fdfd1e469558a599 2024-11-27T13:40:46,485 INFO [StoreOpener-d64103cc558d5243fdfd1e469558a599-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d64103cc558d5243fdfd1e469558a599 columnFamilyName family 2024-11-27T13:40:46,485 DEBUG [StoreOpener-d64103cc558d5243fdfd1e469558a599-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T13:40:46,485 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=61 2024-11-27T13:40:46,486 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=29b1b1ba230f,38845,1732714839599, table=hbase:meta, region=1588230740 2024-11-27T13:40:46,486 INFO [StoreOpener-d64103cc558d5243fdfd1e469558a599-1 {}] regionserver.HStore(327): Store=d64103cc558d5243fdfd1e469558a599/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T13:40:46,486 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 29b1b1ba230f,38845,1732714839599, state=OPEN 2024-11-27T13:40:46,486 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1038): replaying wal for d64103cc558d5243fdfd1e469558a599 2024-11-27T13:40:46,487 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/d64103cc558d5243fdfd1e469558a599 2024-11-27T13:40:46,488 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/d64103cc558d5243fdfd1e469558a599 2024-11-27T13:40:46,488 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1048): stopping wal replay for d64103cc558d5243fdfd1e469558a599 2024-11-27T13:40:46,489 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1060): Cleaning up temporary data for d64103cc558d5243fdfd1e469558a599 2024-11-27T13:40:46,490 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1093): writing seq id for d64103cc558d5243fdfd1e469558a599 2024-11-27T13:40:46,491 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1114): Opened d64103cc558d5243fdfd1e469558a599; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65670403, jitterRate=-0.021434739232063293}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T13:40:46,492 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d64103cc558d5243fdfd1e469558a599 2024-11-27T13:40:46,492 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1006): Region open journal for d64103cc558d5243fdfd1e469558a599: Running coprocessor pre-open hook at 1732714846482Writing region info on filesystem at 1732714846482Initializing all the Stores at 1732714846483 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732714846483Cleaning up temporary data from old regions at 1732714846489 (+6 ms)Running coprocessor post-open hooks at 1732714846492 (+3 ms)Region opened successfully at 1732714846492 2024-11-27T13:40:46,493 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1732714842283.d64103cc558d5243fdfd1e469558a599., pid=73, masterSystemTime=1732714846476 2024-11-27T13:40:46,493 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35019 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-27T13:40:46,507 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1017c8633370001, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:46,507 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35019-0x1017c8633370000, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:46,507 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44709-0x1017c8633370003, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:46,507 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:46,507 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:46,507 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:46,508 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN to state=OFFLINE, location=null, table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b to restore RIT 2024-11-27T13:40:46,509 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN to state=OFFLINE, location=null, table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045 to restore RIT 2024-11-27T13:40:46,509 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN to state=OFFLINE, location=null, table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8 to restore RIT 2024-11-27T13:40:46,509 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f0e25bfe9d22339132aab7b2f16f09, ASSIGN to state=OFFLINE, location=null, table=test, region=f5f0e25bfe9d22339132aab7b2f16f09 to restore RIT 2024-11-27T13:40:46,509 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e7a7c9c02e40375cd974ffe51b6da75b, ASSIGN to state=OFFLINE, location=null, table=test, region=e7a7c9c02e40375cd974ffe51b6da75b to restore RIT 2024-11-27T13:40:46,509 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de4deb0c16ab89c9c56fc8912227f1c5, ASSIGN to state=OFFLINE, location=null, table=test, region=de4deb0c16ab89c9c56fc8912227f1c5 to restore RIT 2024-11-27T13:40:46,509 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d64103cc558d5243fdfd1e469558a599, ASSIGN to state=OFFLINE, location=null, table=test, region=d64103cc558d5243fdfd1e469558a599 to restore RIT 2024-11-27T13:40:46,509 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cccf7ce6a567350b0ae162454b0a9379, ASSIGN to state=OFFLINE, location=null, table=test, region=cccf7ce6a567350b0ae162454b0a9379 to restore RIT 2024-11-27T13:40:46,510 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732714876510 2024-11-27T13:40:46,510 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T13:40:46,512 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-27T13:40:46,513 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-27T13:40:46,513 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "29b1b1ba230f" port: 44709 start_code: 1732714839765 } transition { transition_code: OPENED region_info { region_id: 1732714842283 table_name { namespace: "default" qualifier: "test" } start_key: "yyy" end_key: "zzz" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 73 initiating_master_active_time: 1732714839871 }; retry (#0) after 1005ms delay (Master is coming online...). org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-27T13:40:46,515 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-27T13:40:46,515 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=78, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9ee34ecbedcb7dfdabcb99a67ac21045, server=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,515 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T13:40:46,515 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=78, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9ee34ecbedcb7dfdabcb99a67ac21045, server=29b1b1ba230f,44709,1732714839765 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-27T13:40:46,538 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51060408ec4ea38afe378df89549aeae, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,538 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b6b183b9bc56b2bc4dc6f8968ee01c8, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,539 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0aef6a7e4e138b1b9d7f4aea6971acc6, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,539 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49523b5c318014c3e33791ea6f5e308d, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,539 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4d1ad27263331aa341edfa82a041de56, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,539 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fc6390c5b3b9356bc4885792f770cf2, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,540 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85dc78011fc464d1446075ad14a217de, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,540 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ee34ecbedcb7dfdabcb99a67ac21045, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,540 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dcf91d39846df82f849110c861f2a7d0, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,540 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7a7c9c02e40375cd974ffe51b6da75b, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,541 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eeadb5717c90d7f7a33c12d4883bf2b0, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,541 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a9a679b7eb3754995869ddbc90163b1c, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,541 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cccf7ce6a567350b0ae162454b0a9379, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,541 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d095bb01efaabc5bf5541d373d5419fa, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,542 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c46624524e53a899138ac5a715a35d6, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,542 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de4deb0c16ab89c9c56fc8912227f1c5, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,542 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b448045aa14a354ef4bc74b08978138, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,542 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8700ed52628a2e71e47ec460999f2612, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,543 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5f0e25bfe9d22339132aab7b2f16f09, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,543 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=922622f55d3d471d091912fe46d7c8d8, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,543 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=457728e6ba667138202e8a0dfa04b5b7, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,543 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4f747499d4a9438c3cddbfb1ead0d2b, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,544 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed774066130adf09502c61d07d72f5ab, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,544 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c6bad09b1898b3a3810c7bfeb938da4, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,544 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d64103cc558d5243fdfd1e469558a599, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,544 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d41565a29ddd5c9153a0f41aba598ca, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,544 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-27T13:40:46,544 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732714906544 2024-11-27T13:40:46,545 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732714966545 2024-11-27T13:40:46,545 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 34 msec 2024-11-27T13:40:46,545 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-27T13:40:46,545 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T13:40:46,545 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-27T13:40:46,545 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T13:40:46,545 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-27T13:40:46,545 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-27T13:40:46,545 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-27T13:40:46,546 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T13:40:46,547 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-27T13:40:46,552 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-27T13:40:46,552 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,552 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T13:40:46,552 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,552 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T13:40:46,552 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,552 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T13:40:46,552 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,553 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T13:40:46,553 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,553 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T13:40:46,553 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,553 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T13:40:46,553 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,553 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T13:40:46,553 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,553 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T13:40:46,553 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T13:40:46,553 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,553 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T13:40:46,553 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T13:40:46,553 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T13:40:46,553 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T13:40:46,554 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-27T13:40:46,556 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35019 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-27T13:40:46,558 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true 2024-11-27T13:40:46,558 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,558 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,559 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T13:40:46,559 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:46,559 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,559 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,559 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN 2024-11-27T13:40:46,559 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN 2024-11-27T13:40:46,559 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN 2024-11-27T13:40:46,559 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f0e25bfe9d22339132aab7b2f16f09, ASSIGN 2024-11-27T13:40:46,559 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e7a7c9c02e40375cd974ffe51b6da75b, ASSIGN 2024-11-27T13:40:46,559 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de4deb0c16ab89c9c56fc8912227f1c5, ASSIGN 2024-11-27T13:40:46,559 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d64103cc558d5243fdfd1e469558a599, ASSIGN 2024-11-27T13:40:46,559 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cccf7ce6a567350b0ae162454b0a9379, ASSIGN 2024-11-27T13:40:46,559 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b6b183b9bc56b2bc4dc6f8968ee01c8, server=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:46,559 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure d64103cc558d5243fdfd1e469558a599, server=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,559 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5f0e25bfe9d22339132aab7b2f16f09, server=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,559 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:46,559 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure e7a7c9c02e40375cd974ffe51b6da75b, server=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:46,559 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure c4f747499d4a9438c3cddbfb1ead0d2b, server=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:46,559 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9ee34ecbedcb7dfdabcb99a67ac21045, server=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,559 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,560 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,560 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,560 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,560 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=29b1b1ba230f,38845,1732714839599 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,560 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,560 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,560 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,560 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN 2024-11-27T13:40:46,560 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,560 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN 2024-11-27T13:40:46,560 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,560 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN 2024-11-27T13:40:46,560 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f0e25bfe9d22339132aab7b2f16f09, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,560 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f0e25bfe9d22339132aab7b2f16f09, ASSIGN 2024-11-27T13:40:46,560 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e7a7c9c02e40375cd974ffe51b6da75b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,560 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e7a7c9c02e40375cd974ffe51b6da75b, ASSIGN 2024-11-27T13:40:46,560 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de4deb0c16ab89c9c56fc8912227f1c5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,560 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de4deb0c16ab89c9c56fc8912227f1c5, ASSIGN 2024-11-27T13:40:46,560 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d64103cc558d5243fdfd1e469558a599, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,560 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d64103cc558d5243fdfd1e469558a599, ASSIGN 2024-11-27T13:40:46,560 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cccf7ce6a567350b0ae162454b0a9379, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,560 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cccf7ce6a567350b0ae162454b0a9379, ASSIGN 2024-11-27T13:40:46,560 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b6b183b9bc56b2bc4dc6f8968ee01c8, server=29b1b1ba230f,38845,1732714839599 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,560 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure d64103cc558d5243fdfd1e469558a599, server=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,560 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5f0e25bfe9d22339132aab7b2f16f09, server=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,560 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,560 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure e7a7c9c02e40375cd974ffe51b6da75b, server=29b1b1ba230f,38845,1732714839599 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,560 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure c4f747499d4a9438c3cddbfb1ead0d2b, server=29b1b1ba230f,38845,1732714839599 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,561 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9ee34ecbedcb7dfdabcb99a67ac21045, server=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,561 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 13 msec 2024-11-27T13:40:46,562 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=61 2024-11-27T13:40:46,562 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=29b1b1ba230f,38845,1732714839599, table=hbase:meta, region=1588230740 2024-11-27T13:40:46,562 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 29b1b1ba230f,38845,1732714839599, state=OPEN 2024-11-27T13:40:46,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44709-0x1017c8633370003, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:46,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1017c8633370001, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:46,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35019-0x1017c8633370000, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:46,573 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:46,573 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:46,574 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:46,574 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN to state=OFFLINE, location=null, table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b to restore RIT 2024-11-27T13:40:46,575 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN to state=OFFLINE, location=null, table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045 to restore RIT 2024-11-27T13:40:46,575 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN to state=OFFLINE, location=null, table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8 to restore RIT 2024-11-27T13:40:46,575 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f0e25bfe9d22339132aab7b2f16f09, ASSIGN to state=OFFLINE, location=null, table=test, region=f5f0e25bfe9d22339132aab7b2f16f09 to restore RIT 2024-11-27T13:40:46,575 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e7a7c9c02e40375cd974ffe51b6da75b, ASSIGN to state=OFFLINE, location=null, table=test, region=e7a7c9c02e40375cd974ffe51b6da75b to restore RIT 2024-11-27T13:40:46,575 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de4deb0c16ab89c9c56fc8912227f1c5, ASSIGN to state=OFFLINE, location=null, table=test, region=de4deb0c16ab89c9c56fc8912227f1c5 to restore RIT 2024-11-27T13:40:46,575 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d64103cc558d5243fdfd1e469558a599, ASSIGN to state=OFFLINE, location=null, table=test, region=d64103cc558d5243fdfd1e469558a599 to restore RIT 2024-11-27T13:40:46,575 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cccf7ce6a567350b0ae162454b0a9379, ASSIGN to state=OFFLINE, location=null, table=test, region=cccf7ce6a567350b0ae162454b0a9379 to restore RIT 2024-11-27T13:40:46,575 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732714876575 2024-11-27T13:40:46,575 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T13:40:46,575 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-27T13:40:46,577 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-27T13:40:46,578 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-27T13:40:46,579 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=77, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure c4f747499d4a9438c3cddbfb1ead0d2b, server=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:46,579 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T13:40:46,579 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=77, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure c4f747499d4a9438c3cddbfb1ead0d2b, server=29b1b1ba230f,38845,1732714839599 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-27T13:40:46,585 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51060408ec4ea38afe378df89549aeae, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,585 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b6b183b9bc56b2bc4dc6f8968ee01c8, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,586 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0aef6a7e4e138b1b9d7f4aea6971acc6, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,586 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49523b5c318014c3e33791ea6f5e308d, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,586 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4d1ad27263331aa341edfa82a041de56, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,586 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fc6390c5b3b9356bc4885792f770cf2, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85dc78011fc464d1446075ad14a217de, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ee34ecbedcb7dfdabcb99a67ac21045, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dcf91d39846df82f849110c861f2a7d0, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7a7c9c02e40375cd974ffe51b6da75b, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eeadb5717c90d7f7a33c12d4883bf2b0, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a9a679b7eb3754995869ddbc90163b1c, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,588 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cccf7ce6a567350b0ae162454b0a9379, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,588 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d095bb01efaabc5bf5541d373d5419fa, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,588 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c46624524e53a899138ac5a715a35d6, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,588 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de4deb0c16ab89c9c56fc8912227f1c5, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,588 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b448045aa14a354ef4bc74b08978138, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,589 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8700ed52628a2e71e47ec460999f2612, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,589 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5f0e25bfe9d22339132aab7b2f16f09, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,589 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=922622f55d3d471d091912fe46d7c8d8, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,589 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=457728e6ba667138202e8a0dfa04b5b7, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,589 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4f747499d4a9438c3cddbfb1ead0d2b, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,589 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed774066130adf09502c61d07d72f5ab, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,590 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c6bad09b1898b3a3810c7bfeb938da4, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,590 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d64103cc558d5243fdfd1e469558a599, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,590 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d41565a29ddd5c9153a0f41aba598ca, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,590 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-27T13:40:46,590 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732714906590 2024-11-27T13:40:46,590 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732714966590 2024-11-27T13:40:46,590 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 14 msec 2024-11-27T13:40:46,590 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-27T13:40:46,590 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T13:40:46,590 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-27T13:40:46,590 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T13:40:46,591 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-27T13:40:46,591 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-27T13:40:46,591 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-27T13:40:46,591 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T13:40:46,592 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-27T13:40:46,596 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-27T13:40:46,596 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,596 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T13:40:46,596 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,596 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,597 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T13:40:46,597 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,597 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T13:40:46,597 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,597 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T13:40:46,597 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,597 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T13:40:46,597 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,597 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T13:40:46,597 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,597 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T13:40:46,597 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T13:40:46,597 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,597 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T13:40:46,597 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T13:40:46,597 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T13:40:46,597 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T13:40:46,598 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-27T13:40:46,601 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true 2024-11-27T13:40:46,601 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,601 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,601 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T13:40:46,601 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:46,601 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,601 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,602 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN 2024-11-27T13:40:46,602 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN 2024-11-27T13:40:46,602 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN 2024-11-27T13:40:46,602 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f0e25bfe9d22339132aab7b2f16f09, ASSIGN 2024-11-27T13:40:46,602 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e7a7c9c02e40375cd974ffe51b6da75b, ASSIGN 2024-11-27T13:40:46,602 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de4deb0c16ab89c9c56fc8912227f1c5, ASSIGN 2024-11-27T13:40:46,602 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d64103cc558d5243fdfd1e469558a599, ASSIGN 2024-11-27T13:40:46,602 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cccf7ce6a567350b0ae162454b0a9379, ASSIGN 2024-11-27T13:40:46,602 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b6b183b9bc56b2bc4dc6f8968ee01c8, server=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:46,602 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure d64103cc558d5243fdfd1e469558a599, server=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,602 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5f0e25bfe9d22339132aab7b2f16f09, server=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,602 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:46,602 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure e7a7c9c02e40375cd974ffe51b6da75b, server=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:46,602 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure c4f747499d4a9438c3cddbfb1ead0d2b, server=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:46,602 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,602 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,602 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,602 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,602 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=29b1b1ba230f,38845,1732714839599 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,602 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,602 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,602 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,602 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN 2024-11-27T13:40:46,603 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,603 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN 2024-11-27T13:40:46,603 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,603 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN 2024-11-27T13:40:46,603 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f0e25bfe9d22339132aab7b2f16f09, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,603 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f0e25bfe9d22339132aab7b2f16f09, ASSIGN 2024-11-27T13:40:46,603 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e7a7c9c02e40375cd974ffe51b6da75b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,603 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e7a7c9c02e40375cd974ffe51b6da75b, ASSIGN 2024-11-27T13:40:46,603 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de4deb0c16ab89c9c56fc8912227f1c5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,603 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de4deb0c16ab89c9c56fc8912227f1c5, ASSIGN 2024-11-27T13:40:46,603 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d64103cc558d5243fdfd1e469558a599, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,603 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d64103cc558d5243fdfd1e469558a599, ASSIGN 2024-11-27T13:40:46,603 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cccf7ce6a567350b0ae162454b0a9379, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,603 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cccf7ce6a567350b0ae162454b0a9379, ASSIGN 2024-11-27T13:40:46,603 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b6b183b9bc56b2bc4dc6f8968ee01c8, server=29b1b1ba230f,38845,1732714839599 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,603 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure d64103cc558d5243fdfd1e469558a599, server=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,603 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5f0e25bfe9d22339132aab7b2f16f09, server=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,603 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,603 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure e7a7c9c02e40375cd974ffe51b6da75b, server=29b1b1ba230f,38845,1732714839599 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,603 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure c4f747499d4a9438c3cddbfb1ead0d2b, server=29b1b1ba230f,38845,1732714839599 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,604 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 11 msec 2024-11-27T13:40:46,605 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=61 2024-11-27T13:40:46,605 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=29b1b1ba230f,38845,1732714839599, table=hbase:meta, region=1588230740 2024-11-27T13:40:46,605 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 29b1b1ba230f,38845,1732714839599, state=OPEN 2024-11-27T13:40:46,615 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1017c8633370001, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:46,615 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44709-0x1017c8633370003, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:46,615 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35019-0x1017c8633370000, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:46,616 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:46,616 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:46,616 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:46,616 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN to state=OFFLINE, location=null, table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b to restore RIT 2024-11-27T13:40:46,616 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN to state=OFFLINE, location=null, table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045 to restore RIT 2024-11-27T13:40:46,616 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN to state=OFFLINE, location=null, table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8 to restore RIT 2024-11-27T13:40:46,616 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f0e25bfe9d22339132aab7b2f16f09, ASSIGN to state=OFFLINE, location=null, table=test, region=f5f0e25bfe9d22339132aab7b2f16f09 to restore RIT 2024-11-27T13:40:46,616 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e7a7c9c02e40375cd974ffe51b6da75b, ASSIGN to state=OFFLINE, location=null, table=test, region=e7a7c9c02e40375cd974ffe51b6da75b to restore RIT 2024-11-27T13:40:46,616 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de4deb0c16ab89c9c56fc8912227f1c5, ASSIGN to state=OFFLINE, location=null, table=test, region=de4deb0c16ab89c9c56fc8912227f1c5 to restore RIT 2024-11-27T13:40:46,617 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d64103cc558d5243fdfd1e469558a599, ASSIGN to state=OFFLINE, location=null, table=test, region=d64103cc558d5243fdfd1e469558a599 to restore RIT 2024-11-27T13:40:46,617 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cccf7ce6a567350b0ae162454b0a9379, ASSIGN to state=OFFLINE, location=null, table=test, region=cccf7ce6a567350b0ae162454b0a9379 to restore RIT 2024-11-27T13:40:46,617 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732714876617 2024-11-27T13:40:46,617 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T13:40:46,617 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-27T13:40:46,619 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-27T13:40:46,620 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-27T13:40:46,620 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure e7a7c9c02e40375cd974ffe51b6da75b, server=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:46,620 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T13:40:46,620 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure e7a7c9c02e40375cd974ffe51b6da75b, server=29b1b1ba230f,38845,1732714839599 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-27T13:40:46,628 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51060408ec4ea38afe378df89549aeae, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,628 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b6b183b9bc56b2bc4dc6f8968ee01c8, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,628 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0aef6a7e4e138b1b9d7f4aea6971acc6, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,629 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49523b5c318014c3e33791ea6f5e308d, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,629 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4d1ad27263331aa341edfa82a041de56, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,629 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fc6390c5b3b9356bc4885792f770cf2, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,629 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85dc78011fc464d1446075ad14a217de, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,629 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ee34ecbedcb7dfdabcb99a67ac21045, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,630 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dcf91d39846df82f849110c861f2a7d0, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,630 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7a7c9c02e40375cd974ffe51b6da75b, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,630 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eeadb5717c90d7f7a33c12d4883bf2b0, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,630 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a9a679b7eb3754995869ddbc90163b1c, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,630 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cccf7ce6a567350b0ae162454b0a9379, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,631 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d095bb01efaabc5bf5541d373d5419fa, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,631 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c46624524e53a899138ac5a715a35d6, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,631 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de4deb0c16ab89c9c56fc8912227f1c5, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,631 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b448045aa14a354ef4bc74b08978138, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,631 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8700ed52628a2e71e47ec460999f2612, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,631 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5f0e25bfe9d22339132aab7b2f16f09, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,632 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=922622f55d3d471d091912fe46d7c8d8, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,632 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=457728e6ba667138202e8a0dfa04b5b7, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,632 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4f747499d4a9438c3cddbfb1ead0d2b, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,632 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed774066130adf09502c61d07d72f5ab, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,632 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c6bad09b1898b3a3810c7bfeb938da4, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,632 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d64103cc558d5243fdfd1e469558a599, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,633 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d41565a29ddd5c9153a0f41aba598ca, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,633 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-27T13:40:46,633 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732714906633 2024-11-27T13:40:46,633 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732714966633 2024-11-27T13:40:46,633 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 15 msec 2024-11-27T13:40:46,633 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-27T13:40:46,633 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T13:40:46,633 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-27T13:40:46,633 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T13:40:46,633 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-27T13:40:46,633 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-27T13:40:46,633 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-27T13:40:46,634 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T13:40:46,634 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-27T13:40:46,639 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-27T13:40:46,639 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,639 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,639 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,639 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T13:40:46,639 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,639 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T13:40:46,639 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,639 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T13:40:46,639 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,639 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T13:40:46,639 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,639 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T13:40:46,639 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,639 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T13:40:46,639 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T13:40:46,639 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,639 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T13:40:46,639 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T13:40:46,640 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T13:40:46,640 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T13:40:46,641 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-27T13:40:46,643 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true 2024-11-27T13:40:46,643 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,644 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,644 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T13:40:46,644 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:46,644 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,644 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,644 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN 2024-11-27T13:40:46,644 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN 2024-11-27T13:40:46,644 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN 2024-11-27T13:40:46,644 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f0e25bfe9d22339132aab7b2f16f09, ASSIGN 2024-11-27T13:40:46,644 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e7a7c9c02e40375cd974ffe51b6da75b, ASSIGN 2024-11-27T13:40:46,644 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de4deb0c16ab89c9c56fc8912227f1c5, ASSIGN 2024-11-27T13:40:46,644 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d64103cc558d5243fdfd1e469558a599, ASSIGN 2024-11-27T13:40:46,644 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cccf7ce6a567350b0ae162454b0a9379, ASSIGN 2024-11-27T13:40:46,644 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b6b183b9bc56b2bc4dc6f8968ee01c8, server=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:46,644 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure d64103cc558d5243fdfd1e469558a599, server=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,644 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5f0e25bfe9d22339132aab7b2f16f09, server=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,644 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:46,644 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure e7a7c9c02e40375cd974ffe51b6da75b, server=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:46,644 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,644 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,644 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,645 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,645 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=29b1b1ba230f,38845,1732714839599 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,645 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,645 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,645 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,645 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN 2024-11-27T13:40:46,645 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,645 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN 2024-11-27T13:40:46,645 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,645 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN 2024-11-27T13:40:46,645 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f0e25bfe9d22339132aab7b2f16f09, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,645 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f0e25bfe9d22339132aab7b2f16f09, ASSIGN 2024-11-27T13:40:46,645 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e7a7c9c02e40375cd974ffe51b6da75b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,645 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e7a7c9c02e40375cd974ffe51b6da75b, ASSIGN 2024-11-27T13:40:46,645 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de4deb0c16ab89c9c56fc8912227f1c5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,645 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de4deb0c16ab89c9c56fc8912227f1c5, ASSIGN 2024-11-27T13:40:46,645 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d64103cc558d5243fdfd1e469558a599, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,645 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d64103cc558d5243fdfd1e469558a599, ASSIGN 2024-11-27T13:40:46,645 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cccf7ce6a567350b0ae162454b0a9379, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,645 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cccf7ce6a567350b0ae162454b0a9379, ASSIGN 2024-11-27T13:40:46,645 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b6b183b9bc56b2bc4dc6f8968ee01c8, server=29b1b1ba230f,38845,1732714839599 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,645 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure d64103cc558d5243fdfd1e469558a599, server=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,645 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5f0e25bfe9d22339132aab7b2f16f09, server=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,645 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,645 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure e7a7c9c02e40375cd974ffe51b6da75b, server=29b1b1ba230f,38845,1732714839599 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,646 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 11 msec 2024-11-27T13:40:46,647 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=61 2024-11-27T13:40:46,647 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=29b1b1ba230f,38845,1732714839599, table=hbase:meta, region=1588230740 2024-11-27T13:40:46,648 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 29b1b1ba230f,38845,1732714839599, state=OPEN 2024-11-27T13:40:46,648 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35019 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-27T13:40:46,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1017c8633370001, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:46,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44709-0x1017c8633370003, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:46,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35019-0x1017c8633370000, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:46,658 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:46,658 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:46,658 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:46,659 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN to state=OFFLINE, location=null, table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b to restore RIT 2024-11-27T13:40:46,659 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN to state=OFFLINE, location=null, table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045 to restore RIT 2024-11-27T13:40:46,659 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN to state=OFFLINE, location=null, table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8 to restore RIT 2024-11-27T13:40:46,659 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f0e25bfe9d22339132aab7b2f16f09, ASSIGN to state=OFFLINE, location=null, table=test, region=f5f0e25bfe9d22339132aab7b2f16f09 to restore RIT 2024-11-27T13:40:46,659 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e7a7c9c02e40375cd974ffe51b6da75b, ASSIGN to state=OFFLINE, location=null, table=test, region=e7a7c9c02e40375cd974ffe51b6da75b to restore RIT 2024-11-27T13:40:46,659 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de4deb0c16ab89c9c56fc8912227f1c5, ASSIGN to state=OFFLINE, location=null, table=test, region=de4deb0c16ab89c9c56fc8912227f1c5 to restore RIT 2024-11-27T13:40:46,659 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d64103cc558d5243fdfd1e469558a599, ASSIGN to state=OFFLINE, location=null, table=test, region=d64103cc558d5243fdfd1e469558a599 to restore RIT 2024-11-27T13:40:46,659 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cccf7ce6a567350b0ae162454b0a9379, ASSIGN to state=OFFLINE, location=null, table=test, region=cccf7ce6a567350b0ae162454b0a9379 to restore RIT 2024-11-27T13:40:46,660 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732714876660 2024-11-27T13:40:46,660 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T13:40:46,660 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-27T13:40:46,662 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-27T13:40:46,668 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-27T13:40:46,668 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=75, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:46,668 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T13:40:46,668 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=75, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-27T13:40:46,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51060408ec4ea38afe378df89549aeae, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b6b183b9bc56b2bc4dc6f8968ee01c8, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0aef6a7e4e138b1b9d7f4aea6971acc6, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49523b5c318014c3e33791ea6f5e308d, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4d1ad27263331aa341edfa82a041de56, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fc6390c5b3b9356bc4885792f770cf2, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,671 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85dc78011fc464d1446075ad14a217de, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,671 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ee34ecbedcb7dfdabcb99a67ac21045, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,671 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dcf91d39846df82f849110c861f2a7d0, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,671 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7a7c9c02e40375cd974ffe51b6da75b, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,671 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eeadb5717c90d7f7a33c12d4883bf2b0, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,672 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a9a679b7eb3754995869ddbc90163b1c, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,672 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cccf7ce6a567350b0ae162454b0a9379, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,672 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d095bb01efaabc5bf5541d373d5419fa, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,672 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c46624524e53a899138ac5a715a35d6, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,672 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de4deb0c16ab89c9c56fc8912227f1c5, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,673 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b448045aa14a354ef4bc74b08978138, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,673 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8700ed52628a2e71e47ec460999f2612, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,673 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5f0e25bfe9d22339132aab7b2f16f09, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,673 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=922622f55d3d471d091912fe46d7c8d8, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,674 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=457728e6ba667138202e8a0dfa04b5b7, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,674 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4f747499d4a9438c3cddbfb1ead0d2b, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,674 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed774066130adf09502c61d07d72f5ab, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,674 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c6bad09b1898b3a3810c7bfeb938da4, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,674 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d64103cc558d5243fdfd1e469558a599, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,675 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d41565a29ddd5c9153a0f41aba598ca, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,675 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-27T13:40:46,675 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732714906675 2024-11-27T13:40:46,675 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732714966675 2024-11-27T13:40:46,675 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 14 msec 2024-11-27T13:40:46,675 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-27T13:40:46,675 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T13:40:46,675 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-27T13:40:46,675 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T13:40:46,676 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-27T13:40:46,676 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-27T13:40:46,676 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-27T13:40:46,676 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T13:40:46,677 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-27T13:40:46,682 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-27T13:40:46,682 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,682 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,682 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,682 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T13:40:46,682 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,682 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T13:40:46,682 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,682 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,682 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T13:40:46,682 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,682 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T13:40:46,682 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,682 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T13:40:46,682 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T13:40:46,682 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,683 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T13:40:46,683 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T13:40:46,683 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T13:40:46,683 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T13:40:46,684 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-27T13:40:46,687 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true 2024-11-27T13:40:46,688 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,688 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,688 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T13:40:46,688 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:46,688 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,688 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,688 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN 2024-11-27T13:40:46,688 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN 2024-11-27T13:40:46,688 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN 2024-11-27T13:40:46,688 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f0e25bfe9d22339132aab7b2f16f09, ASSIGN 2024-11-27T13:40:46,688 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e7a7c9c02e40375cd974ffe51b6da75b, ASSIGN 2024-11-27T13:40:46,688 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de4deb0c16ab89c9c56fc8912227f1c5, ASSIGN 2024-11-27T13:40:46,688 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d64103cc558d5243fdfd1e469558a599, ASSIGN 2024-11-27T13:40:46,688 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cccf7ce6a567350b0ae162454b0a9379, ASSIGN 2024-11-27T13:40:46,689 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b6b183b9bc56b2bc4dc6f8968ee01c8, server=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:46,689 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure d64103cc558d5243fdfd1e469558a599, server=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,689 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5f0e25bfe9d22339132aab7b2f16f09, server=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,689 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:46,689 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,689 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,689 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,689 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,689 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=29b1b1ba230f,38845,1732714839599 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,689 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,689 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,689 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,689 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN 2024-11-27T13:40:46,689 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,689 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN 2024-11-27T13:40:46,689 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,689 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN 2024-11-27T13:40:46,689 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f0e25bfe9d22339132aab7b2f16f09, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,689 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f0e25bfe9d22339132aab7b2f16f09, ASSIGN 2024-11-27T13:40:46,690 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e7a7c9c02e40375cd974ffe51b6da75b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,690 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e7a7c9c02e40375cd974ffe51b6da75b, ASSIGN 2024-11-27T13:40:46,690 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de4deb0c16ab89c9c56fc8912227f1c5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,690 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de4deb0c16ab89c9c56fc8912227f1c5, ASSIGN 2024-11-27T13:40:46,690 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d64103cc558d5243fdfd1e469558a599, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,690 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d64103cc558d5243fdfd1e469558a599, ASSIGN 2024-11-27T13:40:46,690 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cccf7ce6a567350b0ae162454b0a9379, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,690 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cccf7ce6a567350b0ae162454b0a9379, ASSIGN 2024-11-27T13:40:46,690 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b6b183b9bc56b2bc4dc6f8968ee01c8, server=29b1b1ba230f,38845,1732714839599 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,690 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure d64103cc558d5243fdfd1e469558a599, server=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,690 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5f0e25bfe9d22339132aab7b2f16f09, server=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,690 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,690 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 13 msec 2024-11-27T13:40:46,691 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=61 2024-11-27T13:40:46,692 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=29b1b1ba230f,38845,1732714839599, table=hbase:meta, region=1588230740 2024-11-27T13:40:46,692 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 29b1b1ba230f,38845,1732714839599, state=OPEN 2024-11-27T13:40:46,699 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35019-0x1017c8633370000, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:46,699 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1017c8633370001, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:46,699 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44709-0x1017c8633370003, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:46,700 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:46,700 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:46,700 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:46,700 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN to state=OFFLINE, location=null, table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b to restore RIT 2024-11-27T13:40:46,700 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN to state=OFFLINE, location=null, table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045 to restore RIT 2024-11-27T13:40:46,701 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN to state=OFFLINE, location=null, table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8 to restore RIT 2024-11-27T13:40:46,701 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f0e25bfe9d22339132aab7b2f16f09, ASSIGN to state=OFFLINE, location=null, table=test, region=f5f0e25bfe9d22339132aab7b2f16f09 to restore RIT 2024-11-27T13:40:46,701 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e7a7c9c02e40375cd974ffe51b6da75b, ASSIGN to state=OFFLINE, location=null, table=test, region=e7a7c9c02e40375cd974ffe51b6da75b to restore RIT 2024-11-27T13:40:46,701 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de4deb0c16ab89c9c56fc8912227f1c5, ASSIGN to state=OFFLINE, location=null, table=test, region=de4deb0c16ab89c9c56fc8912227f1c5 to restore RIT 2024-11-27T13:40:46,701 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d64103cc558d5243fdfd1e469558a599, ASSIGN to state=OFFLINE, location=null, table=test, region=d64103cc558d5243fdfd1e469558a599 to restore RIT 2024-11-27T13:40:46,701 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cccf7ce6a567350b0ae162454b0a9379, ASSIGN to state=OFFLINE, location=null, table=test, region=cccf7ce6a567350b0ae162454b0a9379 to restore RIT 2024-11-27T13:40:46,701 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732714876701 2024-11-27T13:40:46,701 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T13:40:46,701 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-27T13:40:46,703 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-27T13:40:46,704 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-27T13:40:46,704 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5f0e25bfe9d22339132aab7b2f16f09, server=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,704 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T13:40:46,705 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5f0e25bfe9d22339132aab7b2f16f09, server=29b1b1ba230f,44709,1732714839765 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-27T13:40:46,709 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51060408ec4ea38afe378df89549aeae, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,710 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b6b183b9bc56b2bc4dc6f8968ee01c8, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,710 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0aef6a7e4e138b1b9d7f4aea6971acc6, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,710 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49523b5c318014c3e33791ea6f5e308d, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,710 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4d1ad27263331aa341edfa82a041de56, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,711 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fc6390c5b3b9356bc4885792f770cf2, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,711 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85dc78011fc464d1446075ad14a217de, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,711 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ee34ecbedcb7dfdabcb99a67ac21045, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,712 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dcf91d39846df82f849110c861f2a7d0, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,712 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7a7c9c02e40375cd974ffe51b6da75b, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,712 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eeadb5717c90d7f7a33c12d4883bf2b0, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,712 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a9a679b7eb3754995869ddbc90163b1c, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,712 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cccf7ce6a567350b0ae162454b0a9379, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,713 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d095bb01efaabc5bf5541d373d5419fa, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,713 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c46624524e53a899138ac5a715a35d6, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,713 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de4deb0c16ab89c9c56fc8912227f1c5, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,713 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b448045aa14a354ef4bc74b08978138, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,714 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8700ed52628a2e71e47ec460999f2612, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,714 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5f0e25bfe9d22339132aab7b2f16f09, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,714 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=922622f55d3d471d091912fe46d7c8d8, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,714 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=457728e6ba667138202e8a0dfa04b5b7, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,715 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4f747499d4a9438c3cddbfb1ead0d2b, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,715 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed774066130adf09502c61d07d72f5ab, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,715 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c6bad09b1898b3a3810c7bfeb938da4, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,715 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d64103cc558d5243fdfd1e469558a599, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,715 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d41565a29ddd5c9153a0f41aba598ca, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,715 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-27T13:40:46,715 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732714906715 2024-11-27T13:40:46,716 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732714966716 2024-11-27T13:40:46,716 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 14 msec 2024-11-27T13:40:46,716 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-27T13:40:46,716 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T13:40:46,716 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-27T13:40:46,716 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T13:40:46,716 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-27T13:40:46,716 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-27T13:40:46,716 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-27T13:40:46,717 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T13:40:46,718 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-27T13:40:46,722 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-27T13:40:46,722 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,722 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,722 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,722 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T13:40:46,722 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,722 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T13:40:46,722 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,722 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,722 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,722 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T13:40:46,723 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,723 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T13:40:46,723 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T13:40:46,723 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,723 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T13:40:46,723 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T13:40:46,723 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T13:40:46,723 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T13:40:46,724 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-27T13:40:46,727 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true 2024-11-27T13:40:46,727 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,727 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,727 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T13:40:46,727 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:46,727 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,727 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,727 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN 2024-11-27T13:40:46,727 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN 2024-11-27T13:40:46,727 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN 2024-11-27T13:40:46,727 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f0e25bfe9d22339132aab7b2f16f09, ASSIGN 2024-11-27T13:40:46,727 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e7a7c9c02e40375cd974ffe51b6da75b, ASSIGN 2024-11-27T13:40:46,727 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de4deb0c16ab89c9c56fc8912227f1c5, ASSIGN 2024-11-27T13:40:46,728 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d64103cc558d5243fdfd1e469558a599, ASSIGN 2024-11-27T13:40:46,728 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cccf7ce6a567350b0ae162454b0a9379, ASSIGN 2024-11-27T13:40:46,728 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b6b183b9bc56b2bc4dc6f8968ee01c8, server=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:46,728 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure d64103cc558d5243fdfd1e469558a599, server=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,728 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5f0e25bfe9d22339132aab7b2f16f09, server=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,728 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,728 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,728 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,728 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,728 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=29b1b1ba230f,38845,1732714839599 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,728 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,728 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,728 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,728 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN 2024-11-27T13:40:46,729 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,729 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN 2024-11-27T13:40:46,729 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,729 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN 2024-11-27T13:40:46,729 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f0e25bfe9d22339132aab7b2f16f09, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,729 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f0e25bfe9d22339132aab7b2f16f09, ASSIGN 2024-11-27T13:40:46,729 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e7a7c9c02e40375cd974ffe51b6da75b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,729 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e7a7c9c02e40375cd974ffe51b6da75b, ASSIGN 2024-11-27T13:40:46,729 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de4deb0c16ab89c9c56fc8912227f1c5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,729 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de4deb0c16ab89c9c56fc8912227f1c5, ASSIGN 2024-11-27T13:40:46,729 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d64103cc558d5243fdfd1e469558a599, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,729 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d64103cc558d5243fdfd1e469558a599, ASSIGN 2024-11-27T13:40:46,729 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cccf7ce6a567350b0ae162454b0a9379, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,729 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cccf7ce6a567350b0ae162454b0a9379, ASSIGN 2024-11-27T13:40:46,729 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b6b183b9bc56b2bc4dc6f8968ee01c8, server=29b1b1ba230f,38845,1732714839599 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,729 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure d64103cc558d5243fdfd1e469558a599, server=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,729 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5f0e25bfe9d22339132aab7b2f16f09, server=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,729 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 11 msec 2024-11-27T13:40:46,730 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=61 2024-11-27T13:40:46,730 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=29b1b1ba230f,38845,1732714839599, table=hbase:meta, region=1588230740 2024-11-27T13:40:46,731 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 29b1b1ba230f,38845,1732714839599, state=OPEN 2024-11-27T13:40:46,742 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44709-0x1017c8633370003, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:46,742 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1017c8633370001, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:46,742 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35019-0x1017c8633370000, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:46,742 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:46,742 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:46,742 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:46,742 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN to state=OFFLINE, location=null, table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b to restore RIT 2024-11-27T13:40:46,743 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN to state=OFFLINE, location=null, table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045 to restore RIT 2024-11-27T13:40:46,743 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN to state=OFFLINE, location=null, table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8 to restore RIT 2024-11-27T13:40:46,743 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f0e25bfe9d22339132aab7b2f16f09, ASSIGN to state=OFFLINE, location=null, table=test, region=f5f0e25bfe9d22339132aab7b2f16f09 to restore RIT 2024-11-27T13:40:46,743 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e7a7c9c02e40375cd974ffe51b6da75b, ASSIGN to state=OFFLINE, location=null, table=test, region=e7a7c9c02e40375cd974ffe51b6da75b to restore RIT 2024-11-27T13:40:46,743 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de4deb0c16ab89c9c56fc8912227f1c5, ASSIGN to state=OFFLINE, location=null, table=test, region=de4deb0c16ab89c9c56fc8912227f1c5 to restore RIT 2024-11-27T13:40:46,743 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d64103cc558d5243fdfd1e469558a599, ASSIGN to state=OFFLINE, location=null, table=test, region=d64103cc558d5243fdfd1e469558a599 to restore RIT 2024-11-27T13:40:46,743 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cccf7ce6a567350b0ae162454b0a9379, ASSIGN to state=OFFLINE, location=null, table=test, region=cccf7ce6a567350b0ae162454b0a9379 to restore RIT 2024-11-27T13:40:46,743 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732714876743 2024-11-27T13:40:46,743 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T13:40:46,743 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-27T13:40:46,744 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-27T13:40:46,746 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-27T13:40:46,746 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=73, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure d64103cc558d5243fdfd1e469558a599, server=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,746 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T13:40:46,746 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=73, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure d64103cc558d5243fdfd1e469558a599, server=29b1b1ba230f,44709,1732714839765 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-27T13:40:46,749 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51060408ec4ea38afe378df89549aeae, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,749 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b6b183b9bc56b2bc4dc6f8968ee01c8, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,749 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0aef6a7e4e138b1b9d7f4aea6971acc6, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,750 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49523b5c318014c3e33791ea6f5e308d, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,750 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4d1ad27263331aa341edfa82a041de56, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,750 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fc6390c5b3b9356bc4885792f770cf2, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,750 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85dc78011fc464d1446075ad14a217de, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,750 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ee34ecbedcb7dfdabcb99a67ac21045, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,751 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dcf91d39846df82f849110c861f2a7d0, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,751 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7a7c9c02e40375cd974ffe51b6da75b, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,751 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eeadb5717c90d7f7a33c12d4883bf2b0, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,751 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a9a679b7eb3754995869ddbc90163b1c, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,751 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cccf7ce6a567350b0ae162454b0a9379, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,751 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d095bb01efaabc5bf5541d373d5419fa, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,751 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c46624524e53a899138ac5a715a35d6, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,752 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de4deb0c16ab89c9c56fc8912227f1c5, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,752 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b448045aa14a354ef4bc74b08978138, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,752 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8700ed52628a2e71e47ec460999f2612, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,752 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5f0e25bfe9d22339132aab7b2f16f09, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,752 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=922622f55d3d471d091912fe46d7c8d8, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,753 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=457728e6ba667138202e8a0dfa04b5b7, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,753 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4f747499d4a9438c3cddbfb1ead0d2b, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,753 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed774066130adf09502c61d07d72f5ab, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,753 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c6bad09b1898b3a3810c7bfeb938da4, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,753 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d64103cc558d5243fdfd1e469558a599, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,753 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d41565a29ddd5c9153a0f41aba598ca, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,754 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-27T13:40:46,754 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732714906754 2024-11-27T13:40:46,754 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732714966754 2024-11-27T13:40:46,754 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-11-27T13:40:46,754 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-27T13:40:46,754 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T13:40:46,754 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-27T13:40:46,754 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T13:40:46,754 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-27T13:40:46,754 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-27T13:40:46,754 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-27T13:40:46,754 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T13:40:46,755 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-27T13:40:46,758 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-27T13:40:46,758 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,758 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,758 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,758 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T13:40:46,758 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,758 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,758 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,758 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,758 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T13:40:46,758 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,758 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T13:40:46,759 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T13:40:46,759 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,759 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T13:40:46,759 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T13:40:46,759 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T13:40:46,759 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T13:40:46,759 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35019 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-27T13:40:46,760 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-27T13:40:46,762 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true 2024-11-27T13:40:46,762 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,762 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,762 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T13:40:46,762 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:46,762 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,762 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,762 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN 2024-11-27T13:40:46,762 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN 2024-11-27T13:40:46,762 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN 2024-11-27T13:40:46,762 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f0e25bfe9d22339132aab7b2f16f09, ASSIGN 2024-11-27T13:40:46,762 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e7a7c9c02e40375cd974ffe51b6da75b, ASSIGN 2024-11-27T13:40:46,762 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de4deb0c16ab89c9c56fc8912227f1c5, ASSIGN 2024-11-27T13:40:46,762 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d64103cc558d5243fdfd1e469558a599, ASSIGN 2024-11-27T13:40:46,763 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cccf7ce6a567350b0ae162454b0a9379, ASSIGN 2024-11-27T13:40:46,763 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b6b183b9bc56b2bc4dc6f8968ee01c8, server=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:46,763 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure d64103cc558d5243fdfd1e469558a599, server=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,763 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,763 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,763 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,763 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,763 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=29b1b1ba230f,38845,1732714839599 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,763 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,763 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,763 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,763 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35019 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-27T13:40:46,763 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN 2024-11-27T13:40:46,763 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,763 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN 2024-11-27T13:40:46,763 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,763 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN 2024-11-27T13:40:46,763 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f0e25bfe9d22339132aab7b2f16f09, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,764 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f0e25bfe9d22339132aab7b2f16f09, ASSIGN 2024-11-27T13:40:46,764 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e7a7c9c02e40375cd974ffe51b6da75b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,764 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e7a7c9c02e40375cd974ffe51b6da75b, ASSIGN 2024-11-27T13:40:46,764 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de4deb0c16ab89c9c56fc8912227f1c5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,764 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de4deb0c16ab89c9c56fc8912227f1c5, ASSIGN 2024-11-27T13:40:46,764 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d64103cc558d5243fdfd1e469558a599, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,764 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d64103cc558d5243fdfd1e469558a599, ASSIGN 2024-11-27T13:40:46,764 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cccf7ce6a567350b0ae162454b0a9379, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,764 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cccf7ce6a567350b0ae162454b0a9379, ASSIGN 2024-11-27T13:40:46,764 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b6b183b9bc56b2bc4dc6f8968ee01c8, server=29b1b1ba230f,38845,1732714839599 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,764 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure d64103cc558d5243fdfd1e469558a599, server=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,764 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-11-27T13:40:46,765 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=61 2024-11-27T13:40:46,765 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=29b1b1ba230f,38845,1732714839599, table=hbase:meta, region=1588230740 2024-11-27T13:40:46,765 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 29b1b1ba230f,38845,1732714839599, state=OPEN 2024-11-27T13:40:46,773 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1017c8633370001, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:46,773 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35019-0x1017c8633370000, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:46,773 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44709-0x1017c8633370003, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:46,773 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:46,773 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:46,773 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:46,774 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN to state=OFFLINE, location=null, table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b to restore RIT 2024-11-27T13:40:46,774 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN to state=OFFLINE, location=null, table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045 to restore RIT 2024-11-27T13:40:46,774 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN to state=OFFLINE, location=null, table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8 to restore RIT 2024-11-27T13:40:46,774 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f0e25bfe9d22339132aab7b2f16f09, ASSIGN to state=OFFLINE, location=null, table=test, region=f5f0e25bfe9d22339132aab7b2f16f09 to restore RIT 2024-11-27T13:40:46,774 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e7a7c9c02e40375cd974ffe51b6da75b, ASSIGN to state=OFFLINE, location=null, table=test, region=e7a7c9c02e40375cd974ffe51b6da75b to restore RIT 2024-11-27T13:40:46,774 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de4deb0c16ab89c9c56fc8912227f1c5, ASSIGN to state=OFFLINE, location=null, table=test, region=de4deb0c16ab89c9c56fc8912227f1c5 to restore RIT 2024-11-27T13:40:46,774 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d64103cc558d5243fdfd1e469558a599, ASSIGN to state=OFFLINE, location=null, table=test, region=d64103cc558d5243fdfd1e469558a599 to restore RIT 2024-11-27T13:40:46,774 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cccf7ce6a567350b0ae162454b0a9379, ASSIGN to state=OFFLINE, location=null, table=test, region=cccf7ce6a567350b0ae162454b0a9379 to restore RIT 2024-11-27T13:40:46,775 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732714876775 2024-11-27T13:40:46,775 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T13:40:46,775 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-27T13:40:46,776 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-27T13:40:46,777 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-27T13:40:46,777 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b6b183b9bc56b2bc4dc6f8968ee01c8, server=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:46,777 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T13:40:46,777 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b6b183b9bc56b2bc4dc6f8968ee01c8, server=29b1b1ba230f,38845,1732714839599 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-27T13:40:46,781 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51060408ec4ea38afe378df89549aeae, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,782 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b6b183b9bc56b2bc4dc6f8968ee01c8, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,782 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0aef6a7e4e138b1b9d7f4aea6971acc6, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,782 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49523b5c318014c3e33791ea6f5e308d, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,782 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4d1ad27263331aa341edfa82a041de56, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,782 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fc6390c5b3b9356bc4885792f770cf2, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,782 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85dc78011fc464d1446075ad14a217de, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,782 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ee34ecbedcb7dfdabcb99a67ac21045, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,782 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dcf91d39846df82f849110c861f2a7d0, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,782 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7a7c9c02e40375cd974ffe51b6da75b, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,783 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eeadb5717c90d7f7a33c12d4883bf2b0, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,783 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a9a679b7eb3754995869ddbc90163b1c, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,783 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cccf7ce6a567350b0ae162454b0a9379, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,783 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d095bb01efaabc5bf5541d373d5419fa, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,783 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c46624524e53a899138ac5a715a35d6, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,783 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de4deb0c16ab89c9c56fc8912227f1c5, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,783 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b448045aa14a354ef4bc74b08978138, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,783 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8700ed52628a2e71e47ec460999f2612, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,784 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5f0e25bfe9d22339132aab7b2f16f09, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,784 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=922622f55d3d471d091912fe46d7c8d8, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,784 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=457728e6ba667138202e8a0dfa04b5b7, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,784 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4f747499d4a9438c3cddbfb1ead0d2b, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,784 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed774066130adf09502c61d07d72f5ab, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,784 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c6bad09b1898b3a3810c7bfeb938da4, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,784 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d64103cc558d5243fdfd1e469558a599, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,784 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d41565a29ddd5c9153a0f41aba598ca, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,785 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-27T13:40:46,785 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732714906785 2024-11-27T13:40:46,785 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732714966785 2024-11-27T13:40:46,785 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-27T13:40:46,785 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-27T13:40:46,785 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T13:40:46,785 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-27T13:40:46,785 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T13:40:46,785 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-27T13:40:46,785 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-27T13:40:46,785 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-27T13:40:46,786 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T13:40:46,786 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-27T13:40:46,788 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-27T13:40:46,788 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,788 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,788 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,788 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T13:40:46,788 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,788 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,788 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,788 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,788 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,788 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T13:40:46,788 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T13:40:46,789 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,789 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T13:40:46,789 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T13:40:46,789 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T13:40:46,789 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T13:40:46,790 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-27T13:40:46,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true 2024-11-27T13:40:46,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T13:40:46,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:46,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN 2024-11-27T13:40:46,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN 2024-11-27T13:40:46,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN 2024-11-27T13:40:46,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f0e25bfe9d22339132aab7b2f16f09, ASSIGN 2024-11-27T13:40:46,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e7a7c9c02e40375cd974ffe51b6da75b, ASSIGN 2024-11-27T13:40:46,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de4deb0c16ab89c9c56fc8912227f1c5, ASSIGN 2024-11-27T13:40:46,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d64103cc558d5243fdfd1e469558a599, ASSIGN 2024-11-27T13:40:46,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cccf7ce6a567350b0ae162454b0a9379, ASSIGN 2024-11-27T13:40:46,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b6b183b9bc56b2bc4dc6f8968ee01c8, server=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:46,792 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,792 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,792 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,792 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,792 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=29b1b1ba230f,38845,1732714839599 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,792 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,792 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,793 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,793 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN 2024-11-27T13:40:46,793 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,793 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN 2024-11-27T13:40:46,793 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,793 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN 2024-11-27T13:40:46,793 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f0e25bfe9d22339132aab7b2f16f09, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,793 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f0e25bfe9d22339132aab7b2f16f09, ASSIGN 2024-11-27T13:40:46,793 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e7a7c9c02e40375cd974ffe51b6da75b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,793 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e7a7c9c02e40375cd974ffe51b6da75b, ASSIGN 2024-11-27T13:40:46,793 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de4deb0c16ab89c9c56fc8912227f1c5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,793 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de4deb0c16ab89c9c56fc8912227f1c5, ASSIGN 2024-11-27T13:40:46,793 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d64103cc558d5243fdfd1e469558a599, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,793 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d64103cc558d5243fdfd1e469558a599, ASSIGN 2024-11-27T13:40:46,793 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cccf7ce6a567350b0ae162454b0a9379, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,793 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cccf7ce6a567350b0ae162454b0a9379, ASSIGN 2024-11-27T13:40:46,793 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b6b183b9bc56b2bc4dc6f8968ee01c8, server=29b1b1ba230f,38845,1732714839599 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,793 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-27T13:40:46,794 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=61 2024-11-27T13:40:46,794 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=29b1b1ba230f,38845,1732714839599, table=hbase:meta, region=1588230740 2024-11-27T13:40:46,794 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 29b1b1ba230f,38845,1732714839599, state=OPEN 2024-11-27T13:40:46,805 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35019-0x1017c8633370000, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:46,805 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1017c8633370001, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:46,805 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44709-0x1017c8633370003, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:46,805 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:46,805 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:46,805 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:46,806 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN to state=OFFLINE, location=null, table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b to restore RIT 2024-11-27T13:40:46,806 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN to state=OFFLINE, location=null, table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045 to restore RIT 2024-11-27T13:40:46,806 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN to state=OFFLINE, location=null, table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8 to restore RIT 2024-11-27T13:40:46,806 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f0e25bfe9d22339132aab7b2f16f09, ASSIGN to state=OFFLINE, location=null, table=test, region=f5f0e25bfe9d22339132aab7b2f16f09 to restore RIT 2024-11-27T13:40:46,806 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e7a7c9c02e40375cd974ffe51b6da75b, ASSIGN to state=OFFLINE, location=null, table=test, region=e7a7c9c02e40375cd974ffe51b6da75b to restore RIT 2024-11-27T13:40:46,806 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de4deb0c16ab89c9c56fc8912227f1c5, ASSIGN to state=OFFLINE, location=null, table=test, region=de4deb0c16ab89c9c56fc8912227f1c5 to restore RIT 2024-11-27T13:40:46,806 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d64103cc558d5243fdfd1e469558a599, ASSIGN to state=OFFLINE, location=null, table=test, region=d64103cc558d5243fdfd1e469558a599 to restore RIT 2024-11-27T13:40:46,806 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cccf7ce6a567350b0ae162454b0a9379, ASSIGN to state=OFFLINE, location=null, table=test, region=cccf7ce6a567350b0ae162454b0a9379 to restore RIT 2024-11-27T13:40:46,806 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732714876806 2024-11-27T13:40:46,806 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T13:40:46,806 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-27T13:40:46,808 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-27T13:40:46,809 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-27T13:40:46,809 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cccf7ce6a567350b0ae162454b0a9379, ASSIGN 2024-11-27T13:40:46,809 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T13:40:46,809 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cccf7ce6a567350b0ae162454b0a9379, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-27T13:40:46,811 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51060408ec4ea38afe378df89549aeae, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,812 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b6b183b9bc56b2bc4dc6f8968ee01c8, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,812 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0aef6a7e4e138b1b9d7f4aea6971acc6, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,812 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49523b5c318014c3e33791ea6f5e308d, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,812 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4d1ad27263331aa341edfa82a041de56, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,812 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fc6390c5b3b9356bc4885792f770cf2, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,812 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85dc78011fc464d1446075ad14a217de, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,812 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ee34ecbedcb7dfdabcb99a67ac21045, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,812 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dcf91d39846df82f849110c861f2a7d0, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,813 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7a7c9c02e40375cd974ffe51b6da75b, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,813 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eeadb5717c90d7f7a33c12d4883bf2b0, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,813 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a9a679b7eb3754995869ddbc90163b1c, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,813 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cccf7ce6a567350b0ae162454b0a9379, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,813 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d095bb01efaabc5bf5541d373d5419fa, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,813 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c46624524e53a899138ac5a715a35d6, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,813 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de4deb0c16ab89c9c56fc8912227f1c5, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,814 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b448045aa14a354ef4bc74b08978138, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,814 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8700ed52628a2e71e47ec460999f2612, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,814 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5f0e25bfe9d22339132aab7b2f16f09, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,814 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=922622f55d3d471d091912fe46d7c8d8, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,814 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=457728e6ba667138202e8a0dfa04b5b7, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,814 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4f747499d4a9438c3cddbfb1ead0d2b, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,814 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed774066130adf09502c61d07d72f5ab, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,815 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c6bad09b1898b3a3810c7bfeb938da4, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,815 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d64103cc558d5243fdfd1e469558a599, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,815 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d41565a29ddd5c9153a0f41aba598ca, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,815 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-27T13:40:46,815 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732714906815 2024-11-27T13:40:46,815 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732714966815 2024-11-27T13:40:46,815 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-27T13:40:46,815 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-27T13:40:46,815 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T13:40:46,815 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-27T13:40:46,815 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T13:40:46,815 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-27T13:40:46,815 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-27T13:40:46,816 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-27T13:40:46,816 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T13:40:46,817 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-27T13:40:46,818 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-27T13:40:46,818 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,818 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,818 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,818 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,818 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,818 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,818 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,819 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,819 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T13:40:46,819 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T13:40:46,819 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,819 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T13:40:46,819 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T13:40:46,819 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T13:40:46,819 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T13:40:46,819 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-27T13:40:46,821 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true 2024-11-27T13:40:46,821 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,821 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,821 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T13:40:46,821 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:46,821 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,821 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,821 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN 2024-11-27T13:40:46,821 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN 2024-11-27T13:40:46,821 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN 2024-11-27T13:40:46,821 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f0e25bfe9d22339132aab7b2f16f09, ASSIGN 2024-11-27T13:40:46,821 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e7a7c9c02e40375cd974ffe51b6da75b, ASSIGN 2024-11-27T13:40:46,821 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de4deb0c16ab89c9c56fc8912227f1c5, ASSIGN 2024-11-27T13:40:46,821 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d64103cc558d5243fdfd1e469558a599, ASSIGN 2024-11-27T13:40:46,821 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cccf7ce6a567350b0ae162454b0a9379, ASSIGN 2024-11-27T13:40:46,821 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,821 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,821 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,821 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,821 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=29b1b1ba230f,38845,1732714839599 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,822 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,822 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,822 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,822 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN 2024-11-27T13:40:46,822 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,822 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN 2024-11-27T13:40:46,822 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,822 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN 2024-11-27T13:40:46,822 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f0e25bfe9d22339132aab7b2f16f09, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,822 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f0e25bfe9d22339132aab7b2f16f09, ASSIGN 2024-11-27T13:40:46,822 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e7a7c9c02e40375cd974ffe51b6da75b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,822 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e7a7c9c02e40375cd974ffe51b6da75b, ASSIGN 2024-11-27T13:40:46,822 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de4deb0c16ab89c9c56fc8912227f1c5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,822 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de4deb0c16ab89c9c56fc8912227f1c5, ASSIGN 2024-11-27T13:40:46,822 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d64103cc558d5243fdfd1e469558a599, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,822 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d64103cc558d5243fdfd1e469558a599, ASSIGN 2024-11-27T13:40:46,822 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cccf7ce6a567350b0ae162454b0a9379, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,822 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cccf7ce6a567350b0ae162454b0a9379, ASSIGN 2024-11-27T13:40:46,822 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-27T13:40:46,823 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=61 2024-11-27T13:40:46,823 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=29b1b1ba230f,38845,1732714839599, table=hbase:meta, region=1588230740 2024-11-27T13:40:46,823 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 29b1b1ba230f,38845,1732714839599, state=OPEN 2024-11-27T13:40:46,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35019-0x1017c8633370000, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:46,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44709-0x1017c8633370003, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:46,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1017c8633370001, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:46,836 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:46,836 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:46,836 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:46,837 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN to state=OFFLINE, location=null, table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b to restore RIT 2024-11-27T13:40:46,837 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN to state=OFFLINE, location=null, table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045 to restore RIT 2024-11-27T13:40:46,837 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN to state=OFFLINE, location=null, table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8 to restore RIT 2024-11-27T13:40:46,837 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f0e25bfe9d22339132aab7b2f16f09, ASSIGN to state=OFFLINE, location=null, table=test, region=f5f0e25bfe9d22339132aab7b2f16f09 to restore RIT 2024-11-27T13:40:46,837 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e7a7c9c02e40375cd974ffe51b6da75b, ASSIGN to state=OFFLINE, location=null, table=test, region=e7a7c9c02e40375cd974ffe51b6da75b to restore RIT 2024-11-27T13:40:46,837 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de4deb0c16ab89c9c56fc8912227f1c5, ASSIGN to state=OFFLINE, location=null, table=test, region=de4deb0c16ab89c9c56fc8912227f1c5 to restore RIT 2024-11-27T13:40:46,837 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d64103cc558d5243fdfd1e469558a599, ASSIGN to state=OFFLINE, location=null, table=test, region=d64103cc558d5243fdfd1e469558a599 to restore RIT 2024-11-27T13:40:46,837 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cccf7ce6a567350b0ae162454b0a9379, ASSIGN to state=OFFLINE, location=null, table=test, region=cccf7ce6a567350b0ae162454b0a9379 to restore RIT 2024-11-27T13:40:46,838 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732714876838 2024-11-27T13:40:46,838 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T13:40:46,838 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-27T13:40:46,839 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-27T13:40:46,841 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-27T13:40:46,841 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d64103cc558d5243fdfd1e469558a599, ASSIGN 2024-11-27T13:40:46,841 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T13:40:46,841 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d64103cc558d5243fdfd1e469558a599, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-27T13:40:46,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51060408ec4ea38afe378df89549aeae, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b6b183b9bc56b2bc4dc6f8968ee01c8, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0aef6a7e4e138b1b9d7f4aea6971acc6, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49523b5c318014c3e33791ea6f5e308d, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4d1ad27263331aa341edfa82a041de56, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fc6390c5b3b9356bc4885792f770cf2, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85dc78011fc464d1446075ad14a217de, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ee34ecbedcb7dfdabcb99a67ac21045, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dcf91d39846df82f849110c861f2a7d0, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7a7c9c02e40375cd974ffe51b6da75b, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eeadb5717c90d7f7a33c12d4883bf2b0, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a9a679b7eb3754995869ddbc90163b1c, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cccf7ce6a567350b0ae162454b0a9379, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d095bb01efaabc5bf5541d373d5419fa, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c46624524e53a899138ac5a715a35d6, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de4deb0c16ab89c9c56fc8912227f1c5, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,847 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b448045aa14a354ef4bc74b08978138, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,847 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8700ed52628a2e71e47ec460999f2612, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,847 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5f0e25bfe9d22339132aab7b2f16f09, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,847 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=922622f55d3d471d091912fe46d7c8d8, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,847 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=457728e6ba667138202e8a0dfa04b5b7, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,847 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4f747499d4a9438c3cddbfb1ead0d2b, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,847 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed774066130adf09502c61d07d72f5ab, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,848 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c6bad09b1898b3a3810c7bfeb938da4, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,848 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d64103cc558d5243fdfd1e469558a599, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,848 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d41565a29ddd5c9153a0f41aba598ca, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,848 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-27T13:40:46,848 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732714906848 2024-11-27T13:40:46,848 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732714966848 2024-11-27T13:40:46,848 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-11-27T13:40:46,848 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-27T13:40:46,848 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T13:40:46,848 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-27T13:40:46,848 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T13:40:46,849 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-27T13:40:46,849 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-27T13:40:46,849 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-27T13:40:46,849 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T13:40:46,849 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-27T13:40:46,851 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-27T13:40:46,851 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,851 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,851 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,851 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,851 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,852 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,852 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,852 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T13:40:46,852 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T13:40:46,852 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,852 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T13:40:46,852 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T13:40:46,852 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T13:40:46,852 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T13:40:46,853 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-27T13:40:46,854 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true 2024-11-27T13:40:46,854 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,854 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,854 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T13:40:46,854 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:46,854 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,854 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,854 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN 2024-11-27T13:40:46,854 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN 2024-11-27T13:40:46,855 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN 2024-11-27T13:40:46,855 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f0e25bfe9d22339132aab7b2f16f09, ASSIGN 2024-11-27T13:40:46,855 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e7a7c9c02e40375cd974ffe51b6da75b, ASSIGN 2024-11-27T13:40:46,855 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de4deb0c16ab89c9c56fc8912227f1c5, ASSIGN 2024-11-27T13:40:46,855 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d64103cc558d5243fdfd1e469558a599, ASSIGN 2024-11-27T13:40:46,855 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,855 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,855 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,855 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,855 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=29b1b1ba230f,38845,1732714839599 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,855 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,855 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,855 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,855 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN 2024-11-27T13:40:46,855 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,855 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN 2024-11-27T13:40:46,855 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,855 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN 2024-11-27T13:40:46,855 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f0e25bfe9d22339132aab7b2f16f09, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,855 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f0e25bfe9d22339132aab7b2f16f09, ASSIGN 2024-11-27T13:40:46,856 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e7a7c9c02e40375cd974ffe51b6da75b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,856 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e7a7c9c02e40375cd974ffe51b6da75b, ASSIGN 2024-11-27T13:40:46,856 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de4deb0c16ab89c9c56fc8912227f1c5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,856 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de4deb0c16ab89c9c56fc8912227f1c5, ASSIGN 2024-11-27T13:40:46,856 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d64103cc558d5243fdfd1e469558a599, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,856 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d64103cc558d5243fdfd1e469558a599, ASSIGN 2024-11-27T13:40:46,856 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-27T13:40:46,857 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=61 2024-11-27T13:40:46,857 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=29b1b1ba230f,38845,1732714839599, table=hbase:meta, region=1588230740 2024-11-27T13:40:46,857 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 29b1b1ba230f,38845,1732714839599, state=OPEN 2024-11-27T13:40:46,862 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35019 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-27T13:40:46,866 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35019 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-27T13:40:46,868 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44709-0x1017c8633370003, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:46,868 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35019-0x1017c8633370000, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:46,868 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1017c8633370001, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:46,868 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:46,868 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:46,868 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:46,868 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN to state=OFFLINE, location=null, table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b to restore RIT 2024-11-27T13:40:46,868 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN to state=OFFLINE, location=null, table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045 to restore RIT 2024-11-27T13:40:46,869 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN to state=OFFLINE, location=null, table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8 to restore RIT 2024-11-27T13:40:46,869 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f0e25bfe9d22339132aab7b2f16f09, ASSIGN to state=OFFLINE, location=null, table=test, region=f5f0e25bfe9d22339132aab7b2f16f09 to restore RIT 2024-11-27T13:40:46,869 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e7a7c9c02e40375cd974ffe51b6da75b, ASSIGN to state=OFFLINE, location=null, table=test, region=e7a7c9c02e40375cd974ffe51b6da75b to restore RIT 2024-11-27T13:40:46,869 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de4deb0c16ab89c9c56fc8912227f1c5, ASSIGN to state=OFFLINE, location=null, table=test, region=de4deb0c16ab89c9c56fc8912227f1c5 to restore RIT 2024-11-27T13:40:46,869 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d64103cc558d5243fdfd1e469558a599, ASSIGN to state=OFFLINE, location=null, table=test, region=d64103cc558d5243fdfd1e469558a599 to restore RIT 2024-11-27T13:40:46,869 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732714876869 2024-11-27T13:40:46,869 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T13:40:46,869 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-27T13:40:46,871 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-27T13:40:46,872 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-27T13:40:46,872 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de4deb0c16ab89c9c56fc8912227f1c5, ASSIGN 2024-11-27T13:40:46,872 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T13:40:46,872 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de4deb0c16ab89c9c56fc8912227f1c5, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-27T13:40:46,875 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51060408ec4ea38afe378df89549aeae, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,875 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b6b183b9bc56b2bc4dc6f8968ee01c8, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,875 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0aef6a7e4e138b1b9d7f4aea6971acc6, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,875 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49523b5c318014c3e33791ea6f5e308d, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,875 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4d1ad27263331aa341edfa82a041de56, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,875 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fc6390c5b3b9356bc4885792f770cf2, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,876 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85dc78011fc464d1446075ad14a217de, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,876 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ee34ecbedcb7dfdabcb99a67ac21045, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,876 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dcf91d39846df82f849110c861f2a7d0, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,876 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7a7c9c02e40375cd974ffe51b6da75b, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,876 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eeadb5717c90d7f7a33c12d4883bf2b0, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,876 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a9a679b7eb3754995869ddbc90163b1c, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,876 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cccf7ce6a567350b0ae162454b0a9379, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,876 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d095bb01efaabc5bf5541d373d5419fa, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,876 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c46624524e53a899138ac5a715a35d6, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,877 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de4deb0c16ab89c9c56fc8912227f1c5, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,877 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b448045aa14a354ef4bc74b08978138, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,877 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8700ed52628a2e71e47ec460999f2612, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,877 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5f0e25bfe9d22339132aab7b2f16f09, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,877 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=922622f55d3d471d091912fe46d7c8d8, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,877 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=457728e6ba667138202e8a0dfa04b5b7, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,877 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4f747499d4a9438c3cddbfb1ead0d2b, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,877 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed774066130adf09502c61d07d72f5ab, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,877 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c6bad09b1898b3a3810c7bfeb938da4, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,878 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d64103cc558d5243fdfd1e469558a599, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,878 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d41565a29ddd5c9153a0f41aba598ca, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,878 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-27T13:40:46,878 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732714906878 2024-11-27T13:40:46,878 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732714966878 2024-11-27T13:40:46,878 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-27T13:40:46,878 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-27T13:40:46,878 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T13:40:46,878 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-27T13:40:46,878 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T13:40:46,878 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-27T13:40:46,878 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-27T13:40:46,878 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-27T13:40:46,879 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T13:40:46,879 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-27T13:40:46,881 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-27T13:40:46,881 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,881 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,881 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,881 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,881 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,881 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,881 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T13:40:46,881 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T13:40:46,881 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,881 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T13:40:46,881 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T13:40:46,881 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T13:40:46,882 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T13:40:46,882 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-27T13:40:46,883 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true 2024-11-27T13:40:46,883 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,883 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,883 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T13:40:46,884 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:46,884 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,884 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,884 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN 2024-11-27T13:40:46,884 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN 2024-11-27T13:40:46,884 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN 2024-11-27T13:40:46,884 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f0e25bfe9d22339132aab7b2f16f09, ASSIGN 2024-11-27T13:40:46,884 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e7a7c9c02e40375cd974ffe51b6da75b, ASSIGN 2024-11-27T13:40:46,884 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de4deb0c16ab89c9c56fc8912227f1c5, ASSIGN 2024-11-27T13:40:46,884 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,884 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,884 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,884 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,884 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=29b1b1ba230f,38845,1732714839599 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,884 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,884 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,884 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,884 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN 2024-11-27T13:40:46,884 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,884 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN 2024-11-27T13:40:46,884 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,884 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN 2024-11-27T13:40:46,884 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f0e25bfe9d22339132aab7b2f16f09, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,884 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f0e25bfe9d22339132aab7b2f16f09, ASSIGN 2024-11-27T13:40:46,884 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e7a7c9c02e40375cd974ffe51b6da75b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,884 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e7a7c9c02e40375cd974ffe51b6da75b, ASSIGN 2024-11-27T13:40:46,884 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de4deb0c16ab89c9c56fc8912227f1c5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,884 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de4deb0c16ab89c9c56fc8912227f1c5, ASSIGN 2024-11-27T13:40:46,885 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-27T13:40:46,885 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=61 2024-11-27T13:40:46,885 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=29b1b1ba230f,38845,1732714839599, table=hbase:meta, region=1588230740 2024-11-27T13:40:46,886 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 29b1b1ba230f,38845,1732714839599, state=OPEN 2024-11-27T13:40:46,896 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35019-0x1017c8633370000, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:46,896 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44709-0x1017c8633370003, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:46,896 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1017c8633370001, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:46,896 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:46,896 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:46,896 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:46,897 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN to state=OFFLINE, location=null, table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b to restore RIT 2024-11-27T13:40:46,898 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN to state=OFFLINE, location=null, table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045 to restore RIT 2024-11-27T13:40:46,898 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN to state=OFFLINE, location=null, table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8 to restore RIT 2024-11-27T13:40:46,898 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f0e25bfe9d22339132aab7b2f16f09, ASSIGN to state=OFFLINE, location=null, table=test, region=f5f0e25bfe9d22339132aab7b2f16f09 to restore RIT 2024-11-27T13:40:46,898 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e7a7c9c02e40375cd974ffe51b6da75b, ASSIGN to state=OFFLINE, location=null, table=test, region=e7a7c9c02e40375cd974ffe51b6da75b to restore RIT 2024-11-27T13:40:46,898 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de4deb0c16ab89c9c56fc8912227f1c5, ASSIGN to state=OFFLINE, location=null, table=test, region=de4deb0c16ab89c9c56fc8912227f1c5 to restore RIT 2024-11-27T13:40:46,898 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732714876898 2024-11-27T13:40:46,898 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T13:40:46,898 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-27T13:40:46,900 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-27T13:40:46,901 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-27T13:40:46,901 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e7a7c9c02e40375cd974ffe51b6da75b, ASSIGN 2024-11-27T13:40:46,901 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T13:40:46,902 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e7a7c9c02e40375cd974ffe51b6da75b, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-27T13:40:46,905 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51060408ec4ea38afe378df89549aeae, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,905 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b6b183b9bc56b2bc4dc6f8968ee01c8, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,905 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0aef6a7e4e138b1b9d7f4aea6971acc6, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,905 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49523b5c318014c3e33791ea6f5e308d, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,905 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4d1ad27263331aa341edfa82a041de56, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,906 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fc6390c5b3b9356bc4885792f770cf2, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,906 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85dc78011fc464d1446075ad14a217de, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,906 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ee34ecbedcb7dfdabcb99a67ac21045, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,906 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dcf91d39846df82f849110c861f2a7d0, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,906 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7a7c9c02e40375cd974ffe51b6da75b, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,906 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eeadb5717c90d7f7a33c12d4883bf2b0, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,906 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a9a679b7eb3754995869ddbc90163b1c, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,907 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cccf7ce6a567350b0ae162454b0a9379, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,907 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d095bb01efaabc5bf5541d373d5419fa, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,907 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c46624524e53a899138ac5a715a35d6, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,907 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de4deb0c16ab89c9c56fc8912227f1c5, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,907 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b448045aa14a354ef4bc74b08978138, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,907 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8700ed52628a2e71e47ec460999f2612, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,907 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5f0e25bfe9d22339132aab7b2f16f09, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,907 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=922622f55d3d471d091912fe46d7c8d8, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,907 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=457728e6ba667138202e8a0dfa04b5b7, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,908 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4f747499d4a9438c3cddbfb1ead0d2b, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,908 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed774066130adf09502c61d07d72f5ab, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,908 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c6bad09b1898b3a3810c7bfeb938da4, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,908 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d64103cc558d5243fdfd1e469558a599, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,908 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d41565a29ddd5c9153a0f41aba598ca, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,908 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-27T13:40:46,908 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732714906908 2024-11-27T13:40:46,908 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732714966908 2024-11-27T13:40:46,908 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-27T13:40:46,908 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-27T13:40:46,908 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T13:40:46,908 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-27T13:40:46,908 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T13:40:46,909 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-27T13:40:46,909 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-27T13:40:46,909 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-27T13:40:46,909 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T13:40:46,910 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-27T13:40:46,911 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-27T13:40:46,912 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,912 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,912 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,912 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,912 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,912 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T13:40:46,912 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T13:40:46,912 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,912 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T13:40:46,912 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T13:40:46,912 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T13:40:46,912 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T13:40:46,913 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-27T13:40:46,914 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true 2024-11-27T13:40:46,914 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,914 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,914 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T13:40:46,914 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:46,915 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,915 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,915 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN 2024-11-27T13:40:46,915 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN 2024-11-27T13:40:46,915 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN 2024-11-27T13:40:46,915 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f0e25bfe9d22339132aab7b2f16f09, ASSIGN 2024-11-27T13:40:46,915 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e7a7c9c02e40375cd974ffe51b6da75b, ASSIGN 2024-11-27T13:40:46,915 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,915 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,915 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,915 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,915 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=29b1b1ba230f,38845,1732714839599 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,915 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,915 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,915 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,915 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN 2024-11-27T13:40:46,915 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,915 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN 2024-11-27T13:40:46,915 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,915 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN 2024-11-27T13:40:46,915 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f0e25bfe9d22339132aab7b2f16f09, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,915 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f0e25bfe9d22339132aab7b2f16f09, ASSIGN 2024-11-27T13:40:46,915 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e7a7c9c02e40375cd974ffe51b6da75b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,915 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e7a7c9c02e40375cd974ffe51b6da75b, ASSIGN 2024-11-27T13:40:46,916 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-27T13:40:46,916 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=61 2024-11-27T13:40:46,916 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=29b1b1ba230f,38845,1732714839599, table=hbase:meta, region=1588230740 2024-11-27T13:40:46,917 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 29b1b1ba230f,38845,1732714839599, state=OPEN 2024-11-27T13:40:46,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35019-0x1017c8633370000, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:46,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44709-0x1017c8633370003, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:46,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1017c8633370001, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:46,928 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:46,928 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:46,928 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:46,928 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN to state=OFFLINE, location=null, table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b to restore RIT 2024-11-27T13:40:46,928 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN to state=OFFLINE, location=null, table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045 to restore RIT 2024-11-27T13:40:46,928 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN to state=OFFLINE, location=null, table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8 to restore RIT 2024-11-27T13:40:46,928 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f0e25bfe9d22339132aab7b2f16f09, ASSIGN to state=OFFLINE, location=null, table=test, region=f5f0e25bfe9d22339132aab7b2f16f09 to restore RIT 2024-11-27T13:40:46,928 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e7a7c9c02e40375cd974ffe51b6da75b, ASSIGN to state=OFFLINE, location=null, table=test, region=e7a7c9c02e40375cd974ffe51b6da75b to restore RIT 2024-11-27T13:40:46,929 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732714876929 2024-11-27T13:40:46,929 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T13:40:46,929 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-27T13:40:46,931 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-27T13:40:46,932 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-27T13:40:46,932 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f0e25bfe9d22339132aab7b2f16f09, ASSIGN 2024-11-27T13:40:46,932 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T13:40:46,932 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f0e25bfe9d22339132aab7b2f16f09, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-27T13:40:46,936 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51060408ec4ea38afe378df89549aeae, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,936 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b6b183b9bc56b2bc4dc6f8968ee01c8, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,936 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0aef6a7e4e138b1b9d7f4aea6971acc6, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,936 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49523b5c318014c3e33791ea6f5e308d, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,936 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4d1ad27263331aa341edfa82a041de56, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,936 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fc6390c5b3b9356bc4885792f770cf2, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85dc78011fc464d1446075ad14a217de, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ee34ecbedcb7dfdabcb99a67ac21045, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dcf91d39846df82f849110c861f2a7d0, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7a7c9c02e40375cd974ffe51b6da75b, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eeadb5717c90d7f7a33c12d4883bf2b0, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a9a679b7eb3754995869ddbc90163b1c, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cccf7ce6a567350b0ae162454b0a9379, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,938 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d095bb01efaabc5bf5541d373d5419fa, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,938 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c46624524e53a899138ac5a715a35d6, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,938 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de4deb0c16ab89c9c56fc8912227f1c5, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,938 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b448045aa14a354ef4bc74b08978138, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,938 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8700ed52628a2e71e47ec460999f2612, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,938 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5f0e25bfe9d22339132aab7b2f16f09, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,938 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=922622f55d3d471d091912fe46d7c8d8, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,939 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=457728e6ba667138202e8a0dfa04b5b7, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,939 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4f747499d4a9438c3cddbfb1ead0d2b, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,939 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed774066130adf09502c61d07d72f5ab, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,939 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c6bad09b1898b3a3810c7bfeb938da4, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,939 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d64103cc558d5243fdfd1e469558a599, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,939 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d41565a29ddd5c9153a0f41aba598ca, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,939 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-27T13:40:46,940 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732714906939 2024-11-27T13:40:46,940 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732714966940 2024-11-27T13:40:46,940 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-11-27T13:40:46,940 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-27T13:40:46,940 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T13:40:46,940 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-27T13:40:46,940 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T13:40:46,940 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-27T13:40:46,940 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-27T13:40:46,940 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-27T13:40:46,940 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T13:40:46,941 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-27T13:40:46,943 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-27T13:40:46,943 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,943 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,943 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,943 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,943 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T13:40:46,943 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T13:40:46,943 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,943 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T13:40:46,943 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T13:40:46,943 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T13:40:46,943 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T13:40:46,944 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-27T13:40:46,945 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true 2024-11-27T13:40:46,945 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,945 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,945 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T13:40:46,945 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:46,945 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,945 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,945 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN 2024-11-27T13:40:46,946 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN 2024-11-27T13:40:46,946 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN 2024-11-27T13:40:46,946 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f0e25bfe9d22339132aab7b2f16f09, ASSIGN 2024-11-27T13:40:46,946 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,946 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,946 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,946 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,946 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=29b1b1ba230f,38845,1732714839599 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,946 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,946 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,946 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,946 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN 2024-11-27T13:40:46,946 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,946 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN 2024-11-27T13:40:46,946 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,946 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN 2024-11-27T13:40:46,946 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f0e25bfe9d22339132aab7b2f16f09, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,946 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f0e25bfe9d22339132aab7b2f16f09, ASSIGN 2024-11-27T13:40:46,946 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-27T13:40:46,947 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=61 2024-11-27T13:40:46,947 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=29b1b1ba230f,38845,1732714839599, table=hbase:meta, region=1588230740 2024-11-27T13:40:46,947 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 29b1b1ba230f,38845,1732714839599, state=OPEN 2024-11-27T13:40:46,959 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35019-0x1017c8633370000, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:46,959 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1017c8633370001, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:46,959 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44709-0x1017c8633370003, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:46,959 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:46,959 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:46,959 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:46,960 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN to state=OFFLINE, location=null, table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b to restore RIT 2024-11-27T13:40:46,960 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN to state=OFFLINE, location=null, table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045 to restore RIT 2024-11-27T13:40:46,960 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN to state=OFFLINE, location=null, table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8 to restore RIT 2024-11-27T13:40:46,960 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5f0e25bfe9d22339132aab7b2f16f09, ASSIGN to state=OFFLINE, location=null, table=test, region=f5f0e25bfe9d22339132aab7b2f16f09 to restore RIT 2024-11-27T13:40:46,960 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732714876960 2024-11-27T13:40:46,960 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T13:40:46,960 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-27T13:40:46,962 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-27T13:40:46,964 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-27T13:40:46,964 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN 2024-11-27T13:40:46,964 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T13:40:46,964 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-27T13:40:46,967 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51060408ec4ea38afe378df89549aeae, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,967 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b6b183b9bc56b2bc4dc6f8968ee01c8, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,967 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0aef6a7e4e138b1b9d7f4aea6971acc6, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,967 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49523b5c318014c3e33791ea6f5e308d, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,967 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4d1ad27263331aa341edfa82a041de56, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,967 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fc6390c5b3b9356bc4885792f770cf2, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,968 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85dc78011fc464d1446075ad14a217de, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,968 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ee34ecbedcb7dfdabcb99a67ac21045, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,968 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dcf91d39846df82f849110c861f2a7d0, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,968 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7a7c9c02e40375cd974ffe51b6da75b, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,968 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eeadb5717c90d7f7a33c12d4883bf2b0, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,968 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a9a679b7eb3754995869ddbc90163b1c, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,968 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cccf7ce6a567350b0ae162454b0a9379, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,969 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d095bb01efaabc5bf5541d373d5419fa, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,969 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c46624524e53a899138ac5a715a35d6, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,969 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de4deb0c16ab89c9c56fc8912227f1c5, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,969 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b448045aa14a354ef4bc74b08978138, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,969 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8700ed52628a2e71e47ec460999f2612, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,969 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5f0e25bfe9d22339132aab7b2f16f09, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,969 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=922622f55d3d471d091912fe46d7c8d8, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,970 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=457728e6ba667138202e8a0dfa04b5b7, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,970 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4f747499d4a9438c3cddbfb1ead0d2b, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,970 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed774066130adf09502c61d07d72f5ab, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,970 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c6bad09b1898b3a3810c7bfeb938da4, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,971 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d64103cc558d5243fdfd1e469558a599, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,971 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d41565a29ddd5c9153a0f41aba598ca, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,971 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-27T13:40:46,971 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732714906971 2024-11-27T13:40:46,971 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732714966971 2024-11-27T13:40:46,971 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-11-27T13:40:46,971 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-27T13:40:46,971 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T13:40:46,971 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-27T13:40:46,971 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T13:40:46,972 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-27T13:40:46,972 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-27T13:40:46,972 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-27T13:40:46,972 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T13:40:46,973 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-27T13:40:46,975 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-27T13:40:46,975 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,975 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,975 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,975 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T13:40:46,975 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T13:40:46,975 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:46,975 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T13:40:46,975 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T13:40:46,975 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T13:40:46,976 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T13:40:46,976 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-27T13:40:46,977 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true 2024-11-27T13:40:46,977 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,977 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,977 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T13:40:46,977 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:46,977 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,977 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:46,977 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN 2024-11-27T13:40:46,977 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN 2024-11-27T13:40:46,977 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN 2024-11-27T13:40:46,977 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,978 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,978 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,978 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,978 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=29b1b1ba230f,38845,1732714839599 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,978 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,978 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:46,978 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,978 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN 2024-11-27T13:40:46,978 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,978 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN 2024-11-27T13:40:46,978 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:46,978 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN 2024-11-27T13:40:46,978 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-27T13:40:46,979 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=61 2024-11-27T13:40:46,979 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=29b1b1ba230f,38845,1732714839599, table=hbase:meta, region=1588230740 2024-11-27T13:40:46,979 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 29b1b1ba230f,38845,1732714839599, state=OPEN 2024-11-27T13:40:46,990 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35019-0x1017c8633370000, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:46,990 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1017c8633370001, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:46,990 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44709-0x1017c8633370003, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:46,991 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:46,991 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:46,991 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:46,991 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN to state=OFFLINE, location=null, table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b to restore RIT 2024-11-27T13:40:46,992 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN to state=OFFLINE, location=null, table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045 to restore RIT 2024-11-27T13:40:46,992 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8, ASSIGN to state=OFFLINE, location=null, table=test, region=2b6b183b9bc56b2bc4dc6f8968ee01c8 to restore RIT 2024-11-27T13:40:46,992 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732714876992 2024-11-27T13:40:46,992 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T13:40:46,992 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-27T13:40:46,994 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-27T13:40:46,997 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-27T13:40:46,997 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN 2024-11-27T13:40:46,997 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T13:40:46,997 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-27T13:40:46,998 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51060408ec4ea38afe378df89549aeae, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,998 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b6b183b9bc56b2bc4dc6f8968ee01c8, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,999 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0aef6a7e4e138b1b9d7f4aea6971acc6, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,999 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49523b5c318014c3e33791ea6f5e308d, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:46,999 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4d1ad27263331aa341edfa82a041de56, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:46,999 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fc6390c5b3b9356bc4885792f770cf2, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,000 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85dc78011fc464d1446075ad14a217de, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,000 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ee34ecbedcb7dfdabcb99a67ac21045, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,000 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dcf91d39846df82f849110c861f2a7d0, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,000 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7a7c9c02e40375cd974ffe51b6da75b, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,000 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eeadb5717c90d7f7a33c12d4883bf2b0, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,001 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a9a679b7eb3754995869ddbc90163b1c, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,001 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cccf7ce6a567350b0ae162454b0a9379, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,001 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d095bb01efaabc5bf5541d373d5419fa, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,001 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c46624524e53a899138ac5a715a35d6, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,001 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de4deb0c16ab89c9c56fc8912227f1c5, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,002 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b448045aa14a354ef4bc74b08978138, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,002 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8700ed52628a2e71e47ec460999f2612, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,002 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5f0e25bfe9d22339132aab7b2f16f09, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,002 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=922622f55d3d471d091912fe46d7c8d8, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,002 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=457728e6ba667138202e8a0dfa04b5b7, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,002 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4f747499d4a9438c3cddbfb1ead0d2b, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,003 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed774066130adf09502c61d07d72f5ab, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,003 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c6bad09b1898b3a3810c7bfeb938da4, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,003 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d64103cc558d5243fdfd1e469558a599, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,003 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d41565a29ddd5c9153a0f41aba598ca, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,003 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-27T13:40:47,003 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732714907003 2024-11-27T13:40:47,004 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732714967003 2024-11-27T13:40:47,004 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 11 msec 2024-11-27T13:40:47,004 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-27T13:40:47,004 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T13:40:47,004 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-27T13:40:47,004 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T13:40:47,004 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-27T13:40:47,004 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-27T13:40:47,004 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-27T13:40:47,005 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T13:40:47,006 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-27T13:40:47,008 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-27T13:40:47,008 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:47,008 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:47,008 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T13:40:47,008 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T13:40:47,008 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:47,008 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T13:40:47,008 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T13:40:47,008 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T13:40:47,009 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T13:40:47,009 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-27T13:40:47,011 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true 2024-11-27T13:40:47,011 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:47,011 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:47,011 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T13:40:47,011 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:47,011 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:47,011 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:47,011 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN 2024-11-27T13:40:47,011 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN 2024-11-27T13:40:47,011 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:47,011 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:47,011 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:47,011 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:47,011 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=29b1b1ba230f,38845,1732714839599 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:47,011 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:47,011 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:47,011 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:47,011 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN 2024-11-27T13:40:47,012 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:47,012 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN 2024-11-27T13:40:47,012 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-27T13:40:47,013 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=61 2024-11-27T13:40:47,013 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=29b1b1ba230f,38845,1732714839599, table=hbase:meta, region=1588230740 2024-11-27T13:40:47,013 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 29b1b1ba230f,38845,1732714839599, state=OPEN 2024-11-27T13:40:47,056 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44709-0x1017c8633370003, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:47,056 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35019-0x1017c8633370000, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:47,056 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1017c8633370001, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:47,056 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:47,056 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:47,056 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:47,057 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN to state=OFFLINE, location=null, table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b to restore RIT 2024-11-27T13:40:47,057 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045, ASSIGN to state=OFFLINE, location=null, table=test, region=9ee34ecbedcb7dfdabcb99a67ac21045 to restore RIT 2024-11-27T13:40:47,057 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732714877057 2024-11-27T13:40:47,057 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T13:40:47,057 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-27T13:40:47,059 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-27T13:40:47,060 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-27T13:40:47,060 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN 2024-11-27T13:40:47,060 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T13:40:47,060 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-27T13:40:47,062 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51060408ec4ea38afe378df89549aeae, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,062 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b6b183b9bc56b2bc4dc6f8968ee01c8, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,062 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0aef6a7e4e138b1b9d7f4aea6971acc6, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,062 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49523b5c318014c3e33791ea6f5e308d, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,063 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4d1ad27263331aa341edfa82a041de56, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,063 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fc6390c5b3b9356bc4885792f770cf2, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,063 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85dc78011fc464d1446075ad14a217de, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,063 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ee34ecbedcb7dfdabcb99a67ac21045, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,063 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dcf91d39846df82f849110c861f2a7d0, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,063 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7a7c9c02e40375cd974ffe51b6da75b, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,063 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eeadb5717c90d7f7a33c12d4883bf2b0, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,063 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a9a679b7eb3754995869ddbc90163b1c, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,063 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cccf7ce6a567350b0ae162454b0a9379, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,063 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d095bb01efaabc5bf5541d373d5419fa, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,063 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c46624524e53a899138ac5a715a35d6, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,063 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de4deb0c16ab89c9c56fc8912227f1c5, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,064 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b448045aa14a354ef4bc74b08978138, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,064 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8700ed52628a2e71e47ec460999f2612, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,064 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5f0e25bfe9d22339132aab7b2f16f09, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,064 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=922622f55d3d471d091912fe46d7c8d8, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,064 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=457728e6ba667138202e8a0dfa04b5b7, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,064 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4f747499d4a9438c3cddbfb1ead0d2b, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,064 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed774066130adf09502c61d07d72f5ab, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,064 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c6bad09b1898b3a3810c7bfeb938da4, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,064 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d64103cc558d5243fdfd1e469558a599, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,064 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d41565a29ddd5c9153a0f41aba598ca, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,064 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-27T13:40:47,064 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732714907064 2024-11-27T13:40:47,065 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732714967065 2024-11-27T13:40:47,065 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-27T13:40:47,065 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-27T13:40:47,065 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T13:40:47,065 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-27T13:40:47,065 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T13:40:47,065 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-27T13:40:47,065 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-27T13:40:47,065 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-27T13:40:47,065 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T13:40:47,066 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-27T13:40:47,067 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-27T13:40:47,067 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:47,067 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T13:40:47,067 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T13:40:47,067 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:47,067 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T13:40:47,068 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T13:40:47,068 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T13:40:47,068 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T13:40:47,068 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-27T13:40:47,069 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true 2024-11-27T13:40:47,069 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:47,069 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:47,069 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T13:40:47,069 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:47,069 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:47,069 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:47,069 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN 2024-11-27T13:40:47,069 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:47,069 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:47,070 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:47,070 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:47,070 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=29b1b1ba230f,38845,1732714839599 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:47,070 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:47,070 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:47,070 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:47,070 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN 2024-11-27T13:40:47,070 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-27T13:40:47,070 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=61 2024-11-27T13:40:47,071 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=29b1b1ba230f,38845,1732714839599, table=hbase:meta, region=1588230740 2024-11-27T13:40:47,071 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 29b1b1ba230f,38845,1732714839599, state=OPEN 2024-11-27T13:40:47,073 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35019 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-27T13:40:47,073 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35019 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-27T13:40:47,089 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1017c8633370001, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:47,089 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35019-0x1017c8633370000, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:47,089 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44709-0x1017c8633370003, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:47,089 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:47,089 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:47,089 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:47,090 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b, ASSIGN to state=OFFLINE, location=null, table=test, region=c4f747499d4a9438c3cddbfb1ead0d2b to restore RIT 2024-11-27T13:40:47,090 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732714877090 2024-11-27T13:40:47,090 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T13:40:47,090 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-27T13:40:47,092 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-27T13:40:47,093 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-27T13:40:47,093 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:47,093 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T13:40:47,093 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-27T13:40:47,096 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51060408ec4ea38afe378df89549aeae, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,096 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b6b183b9bc56b2bc4dc6f8968ee01c8, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,096 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0aef6a7e4e138b1b9d7f4aea6971acc6, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,096 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49523b5c318014c3e33791ea6f5e308d, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,096 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4d1ad27263331aa341edfa82a041de56, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,097 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fc6390c5b3b9356bc4885792f770cf2, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,097 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85dc78011fc464d1446075ad14a217de, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,097 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ee34ecbedcb7dfdabcb99a67ac21045, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,097 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dcf91d39846df82f849110c861f2a7d0, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,097 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7a7c9c02e40375cd974ffe51b6da75b, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,097 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eeadb5717c90d7f7a33c12d4883bf2b0, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,097 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a9a679b7eb3754995869ddbc90163b1c, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,097 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cccf7ce6a567350b0ae162454b0a9379, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,097 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d095bb01efaabc5bf5541d373d5419fa, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,098 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c46624524e53a899138ac5a715a35d6, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,098 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de4deb0c16ab89c9c56fc8912227f1c5, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,098 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b448045aa14a354ef4bc74b08978138, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,098 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8700ed52628a2e71e47ec460999f2612, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,098 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5f0e25bfe9d22339132aab7b2f16f09, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,098 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=922622f55d3d471d091912fe46d7c8d8, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,098 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=457728e6ba667138202e8a0dfa04b5b7, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,098 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4f747499d4a9438c3cddbfb1ead0d2b, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,098 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed774066130adf09502c61d07d72f5ab, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,098 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c6bad09b1898b3a3810c7bfeb938da4, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,098 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d64103cc558d5243fdfd1e469558a599, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,099 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d41565a29ddd5c9153a0f41aba598ca, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,099 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-27T13:40:47,099 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732714907099 2024-11-27T13:40:47,099 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732714967099 2024-11-27T13:40:47,099 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-27T13:40:47,099 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-27T13:40:47,099 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T13:40:47,099 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-27T13:40:47,099 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T13:40:47,099 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-27T13:40:47,099 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-27T13:40:47,099 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-27T13:40:47,099 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T13:40:47,100 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-27T13:40:47,102 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-27T13:40:47,102 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T13:40:47,102 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T13:40:47,102 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:47,102 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T13:40:47,103 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T13:40:47,103 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T13:40:47,103 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T13:40:47,103 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-27T13:40:47,104 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true 2024-11-27T13:40:47,104 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:47,104 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:47,105 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T13:40:47,105 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:47,105 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:47,105 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:47,105 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:47,105 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:47,105 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:47,105 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:47,105 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=29b1b1ba230f,38845,1732714839599 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:47,105 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:47,105 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:47,105 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-27T13:40:47,106 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=61 2024-11-27T13:40:47,106 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=29b1b1ba230f,38845,1732714839599, table=hbase:meta, region=1588230740 2024-11-27T13:40:47,106 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 29b1b1ba230f,38845,1732714839599, state=OPEN 2024-11-27T13:40:47,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44709-0x1017c8633370003, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:47,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35019-0x1017c8633370000, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:47,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1017c8633370001, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:47,117 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:47,117 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:47,117 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:47,118 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732714877118 2024-11-27T13:40:47,118 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T13:40:47,118 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-27T13:40:47,119 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-27T13:40:47,120 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-27T13:40:47,121 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:47,121 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T13:40:47,121 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-27T13:40:47,123 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51060408ec4ea38afe378df89549aeae, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,123 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b6b183b9bc56b2bc4dc6f8968ee01c8, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,123 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0aef6a7e4e138b1b9d7f4aea6971acc6, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,123 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49523b5c318014c3e33791ea6f5e308d, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,123 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4d1ad27263331aa341edfa82a041de56, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,123 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fc6390c5b3b9356bc4885792f770cf2, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,124 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85dc78011fc464d1446075ad14a217de, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,124 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ee34ecbedcb7dfdabcb99a67ac21045, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,124 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dcf91d39846df82f849110c861f2a7d0, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,124 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7a7c9c02e40375cd974ffe51b6da75b, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,124 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eeadb5717c90d7f7a33c12d4883bf2b0, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,124 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a9a679b7eb3754995869ddbc90163b1c, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,124 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cccf7ce6a567350b0ae162454b0a9379, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,124 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d095bb01efaabc5bf5541d373d5419fa, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,125 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c46624524e53a899138ac5a715a35d6, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,125 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de4deb0c16ab89c9c56fc8912227f1c5, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,125 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b448045aa14a354ef4bc74b08978138, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,125 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8700ed52628a2e71e47ec460999f2612, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,125 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5f0e25bfe9d22339132aab7b2f16f09, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,125 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=922622f55d3d471d091912fe46d7c8d8, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,125 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=457728e6ba667138202e8a0dfa04b5b7, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,125 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4f747499d4a9438c3cddbfb1ead0d2b, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,126 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed774066130adf09502c61d07d72f5ab, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,126 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c6bad09b1898b3a3810c7bfeb938da4, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,126 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d64103cc558d5243fdfd1e469558a599, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,126 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d41565a29ddd5c9153a0f41aba598ca, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,126 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-27T13:40:47,126 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732714907126 2024-11-27T13:40:47,126 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732714967126 2024-11-27T13:40:47,126 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-27T13:40:47,126 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-27T13:40:47,126 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T13:40:47,126 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-27T13:40:47,127 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T13:40:47,127 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-27T13:40:47,127 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-27T13:40:47,127 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-27T13:40:47,127 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T13:40:47,128 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-27T13:40:47,130 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-27T13:40:47,130 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T13:40:47,130 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T13:40:47,130 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:47,130 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T13:40:47,130 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T13:40:47,130 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T13:40:47,131 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-27T13:40:47,132 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true 2024-11-27T13:40:47,132 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:47,132 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:47,132 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T13:40:47,132 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:47,132 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:47,132 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:47,132 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:47,132 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:47,132 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:47,133 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=29b1b1ba230f,38845,1732714839599 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:47,133 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.1732714841272, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:47,133 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-27T13:40:47,133 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=61 2024-11-27T13:40:47,134 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=29b1b1ba230f,38845,1732714839599, table=hbase:meta, region=1588230740 2024-11-27T13:40:47,134 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 29b1b1ba230f,38845,1732714839599, state=OPEN 2024-11-27T13:40:47,141 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44709-0x1017c8633370003, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:47,142 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1017c8633370001, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:47,142 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35019-0x1017c8633370000, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:47,142 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:47,142 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:47,142 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:47,143 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732714877143 2024-11-27T13:40:47,143 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T13:40:47,143 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-27T13:40:47,144 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-27T13:40:47,145 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-27T13:40:47,145 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:47,145 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T13:40:47,145 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=29b1b1ba230f,38845,1732714839599 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-27T13:40:47,147 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51060408ec4ea38afe378df89549aeae, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,148 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b6b183b9bc56b2bc4dc6f8968ee01c8, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,148 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0aef6a7e4e138b1b9d7f4aea6971acc6, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,148 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49523b5c318014c3e33791ea6f5e308d, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,148 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4d1ad27263331aa341edfa82a041de56, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,148 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fc6390c5b3b9356bc4885792f770cf2, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,148 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85dc78011fc464d1446075ad14a217de, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,149 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ee34ecbedcb7dfdabcb99a67ac21045, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,149 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dcf91d39846df82f849110c861f2a7d0, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,149 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7a7c9c02e40375cd974ffe51b6da75b, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,149 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eeadb5717c90d7f7a33c12d4883bf2b0, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,149 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a9a679b7eb3754995869ddbc90163b1c, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,149 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cccf7ce6a567350b0ae162454b0a9379, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,149 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d095bb01efaabc5bf5541d373d5419fa, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,149 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c46624524e53a899138ac5a715a35d6, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de4deb0c16ab89c9c56fc8912227f1c5, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b448045aa14a354ef4bc74b08978138, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8700ed52628a2e71e47ec460999f2612, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5f0e25bfe9d22339132aab7b2f16f09, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=922622f55d3d471d091912fe46d7c8d8, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=457728e6ba667138202e8a0dfa04b5b7, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4f747499d4a9438c3cddbfb1ead0d2b, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed774066130adf09502c61d07d72f5ab, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c6bad09b1898b3a3810c7bfeb938da4, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d64103cc558d5243fdfd1e469558a599, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d41565a29ddd5c9153a0f41aba598ca, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,150 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-27T13:40:47,150 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732714907150 2024-11-27T13:40:47,150 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732714967150 2024-11-27T13:40:47,150 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-27T13:40:47,151 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-27T13:40:47,151 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T13:40:47,151 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-27T13:40:47,151 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T13:40:47,151 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-27T13:40:47,151 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-27T13:40:47,151 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-27T13:40:47,151 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T13:40:47,152 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-27T13:40:47,153 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-27T13:40:47,153 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T13:40:47,153 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T13:40:47,153 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:47,154 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-27T13:40:47,154 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T13:40:47,154 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-27T13:40:47,155 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true 2024-11-27T13:40:47,155 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:47,155 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:47,155 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T13:40:47,155 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:47,155 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:47,155 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:47,155 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:47,155 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:47,155 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=29b1b1ba230f,38845,1732714839599 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:47,155 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-27T13:40:47,156 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=61 2024-11-27T13:40:47,156 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=29b1b1ba230f,38845,1732714839599, table=hbase:meta, region=1588230740 2024-11-27T13:40:47,156 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 29b1b1ba230f,38845,1732714839599, state=OPEN 2024-11-27T13:40:47,163 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1017c8633370001, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:47,163 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44709-0x1017c8633370003, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:47,163 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35019-0x1017c8633370000, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:47,163 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:47,163 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:47,163 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:47,164 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732714877163 2024-11-27T13:40:47,164 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T13:40:47,164 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-27T13:40:47,165 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-27T13:40:47,166 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-27T13:40:47,166 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T13:40:47,166 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T13:40:47,166 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-27T13:40:47,169 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51060408ec4ea38afe378df89549aeae, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,169 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b6b183b9bc56b2bc4dc6f8968ee01c8, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,169 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0aef6a7e4e138b1b9d7f4aea6971acc6, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,169 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49523b5c318014c3e33791ea6f5e308d, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,169 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4d1ad27263331aa341edfa82a041de56, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,169 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fc6390c5b3b9356bc4885792f770cf2, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,169 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85dc78011fc464d1446075ad14a217de, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,169 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ee34ecbedcb7dfdabcb99a67ac21045, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,169 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dcf91d39846df82f849110c861f2a7d0, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,169 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7a7c9c02e40375cd974ffe51b6da75b, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,169 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eeadb5717c90d7f7a33c12d4883bf2b0, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a9a679b7eb3754995869ddbc90163b1c, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cccf7ce6a567350b0ae162454b0a9379, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d095bb01efaabc5bf5541d373d5419fa, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c46624524e53a899138ac5a715a35d6, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de4deb0c16ab89c9c56fc8912227f1c5, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b448045aa14a354ef4bc74b08978138, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8700ed52628a2e71e47ec460999f2612, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5f0e25bfe9d22339132aab7b2f16f09, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=922622f55d3d471d091912fe46d7c8d8, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=457728e6ba667138202e8a0dfa04b5b7, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4f747499d4a9438c3cddbfb1ead0d2b, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed774066130adf09502c61d07d72f5ab, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c6bad09b1898b3a3810c7bfeb938da4, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d64103cc558d5243fdfd1e469558a599, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,171 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d41565a29ddd5c9153a0f41aba598ca, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,171 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-27T13:40:47,171 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732714907171 2024-11-27T13:40:47,171 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732714967171 2024-11-27T13:40:47,171 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-27T13:40:47,171 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-27T13:40:47,171 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T13:40:47,171 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-27T13:40:47,171 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T13:40:47,171 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-27T13:40:47,171 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-27T13:40:47,171 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-27T13:40:47,171 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T13:40:47,172 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-27T13:40:47,174 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-27T13:40:47,174 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T13:40:47,174 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T13:40:47,174 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-27T13:40:47,174 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T13:40:47,175 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-27T13:40:47,175 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true 2024-11-27T13:40:47,175 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:47,176 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:47,176 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T13:40:47,176 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:47,176 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:47,176 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:47,176 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:47,176 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-27T13:40:47,177 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=61 2024-11-27T13:40:47,177 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=29b1b1ba230f,38845,1732714839599, table=hbase:meta, region=1588230740 2024-11-27T13:40:47,177 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 29b1b1ba230f,38845,1732714839599, state=OPEN 2024-11-27T13:40:47,184 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44709-0x1017c8633370003, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:47,184 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35019-0x1017c8633370000, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:47,184 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1017c8633370001, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:47,184 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:47,184 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:47,184 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:47,185 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732714877185 2024-11-27T13:40:47,185 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T13:40:47,185 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-27T13:40:47,187 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-27T13:40:47,190 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-27T13:40:47,190 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:47,190 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T13:40:47,190 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-27T13:40:47,191 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51060408ec4ea38afe378df89549aeae, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,191 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b6b183b9bc56b2bc4dc6f8968ee01c8, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,191 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0aef6a7e4e138b1b9d7f4aea6971acc6, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,191 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49523b5c318014c3e33791ea6f5e308d, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,191 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4d1ad27263331aa341edfa82a041de56, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,191 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fc6390c5b3b9356bc4885792f770cf2, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,191 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85dc78011fc464d1446075ad14a217de, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,191 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ee34ecbedcb7dfdabcb99a67ac21045, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,192 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dcf91d39846df82f849110c861f2a7d0, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,192 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7a7c9c02e40375cd974ffe51b6da75b, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,192 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eeadb5717c90d7f7a33c12d4883bf2b0, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,192 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a9a679b7eb3754995869ddbc90163b1c, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,192 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cccf7ce6a567350b0ae162454b0a9379, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,192 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d095bb01efaabc5bf5541d373d5419fa, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,192 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c46624524e53a899138ac5a715a35d6, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,192 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de4deb0c16ab89c9c56fc8912227f1c5, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,192 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b448045aa14a354ef4bc74b08978138, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,192 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8700ed52628a2e71e47ec460999f2612, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,192 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5f0e25bfe9d22339132aab7b2f16f09, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,193 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=922622f55d3d471d091912fe46d7c8d8, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,193 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=457728e6ba667138202e8a0dfa04b5b7, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,193 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4f747499d4a9438c3cddbfb1ead0d2b, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,193 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed774066130adf09502c61d07d72f5ab, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,193 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c6bad09b1898b3a3810c7bfeb938da4, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,193 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d64103cc558d5243fdfd1e469558a599, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,193 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d41565a29ddd5c9153a0f41aba598ca, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,193 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-27T13:40:47,193 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732714907193 2024-11-27T13:40:47,193 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732714967193 2024-11-27T13:40:47,193 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-27T13:40:47,193 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-27T13:40:47,193 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T13:40:47,193 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-27T13:40:47,193 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T13:40:47,194 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-27T13:40:47,194 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-27T13:40:47,194 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-27T13:40:47,194 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T13:40:47,195 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-27T13:40:47,196 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-27T13:40:47,196 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T13:40:47,196 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-27T13:40:47,196 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T13:40:47,197 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-27T13:40:47,197 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true 2024-11-27T13:40:47,197 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:47,197 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:47,197 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:47,197 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:47,197 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:47,197 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-27T13:40:47,198 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=61 2024-11-27T13:40:47,198 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=29b1b1ba230f,38845,1732714839599, table=hbase:meta, region=1588230740 2024-11-27T13:40:47,198 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 29b1b1ba230f,38845,1732714839599, state=OPEN 2024-11-27T13:40:47,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44709-0x1017c8633370003, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:47,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35019-0x1017c8633370000, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:47,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1017c8633370001, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:47,205 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:47,205 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:47,205 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:47,206 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732714877206 2024-11-27T13:40:47,206 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T13:40:47,206 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-27T13:40:47,207 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-27T13:40:47,208 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-27T13:40:47,209 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:47,209 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T13:40:47,209 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-27T13:40:47,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51060408ec4ea38afe378df89549aeae, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b6b183b9bc56b2bc4dc6f8968ee01c8, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0aef6a7e4e138b1b9d7f4aea6971acc6, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49523b5c318014c3e33791ea6f5e308d, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4d1ad27263331aa341edfa82a041de56, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fc6390c5b3b9356bc4885792f770cf2, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85dc78011fc464d1446075ad14a217de, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,213 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ee34ecbedcb7dfdabcb99a67ac21045, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,213 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dcf91d39846df82f849110c861f2a7d0, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,213 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7a7c9c02e40375cd974ffe51b6da75b, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,213 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eeadb5717c90d7f7a33c12d4883bf2b0, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,213 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a9a679b7eb3754995869ddbc90163b1c, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,213 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cccf7ce6a567350b0ae162454b0a9379, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,213 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d095bb01efaabc5bf5541d373d5419fa, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,213 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c46624524e53a899138ac5a715a35d6, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,214 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de4deb0c16ab89c9c56fc8912227f1c5, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,214 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b448045aa14a354ef4bc74b08978138, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,214 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8700ed52628a2e71e47ec460999f2612, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,214 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5f0e25bfe9d22339132aab7b2f16f09, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,214 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=922622f55d3d471d091912fe46d7c8d8, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,214 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=457728e6ba667138202e8a0dfa04b5b7, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,214 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4f747499d4a9438c3cddbfb1ead0d2b, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,214 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed774066130adf09502c61d07d72f5ab, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,214 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c6bad09b1898b3a3810c7bfeb938da4, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,215 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d64103cc558d5243fdfd1e469558a599, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,215 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d41565a29ddd5c9153a0f41aba598ca, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,215 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-27T13:40:47,215 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732714907215 2024-11-27T13:40:47,215 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732714967215 2024-11-27T13:40:47,215 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-27T13:40:47,215 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-27T13:40:47,215 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T13:40:47,215 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-27T13:40:47,215 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T13:40:47,215 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-27T13:40:47,215 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-27T13:40:47,216 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-27T13:40:47,216 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T13:40:47,217 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-27T13:40:47,218 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-27T13:40:47,219 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-27T13:40:47,219 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T13:40:47,219 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-27T13:40:47,220 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true 2024-11-27T13:40:47,220 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:47,220 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:47,220 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 29b1b1ba230f%2C39411%2C1732714839711.meta.1732714841860.meta, worker=29b1b1ba230f,44709,1732714839765 didn't hold the lock before restarting, skip acquiring lock. 2024-11-27T13:40:47,220 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-27T13:40:47,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=61 2024-11-27T13:40:47,221 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=29b1b1ba230f,38845,1732714839599, table=hbase:meta, region=1588230740 2024-11-27T13:40:47,222 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 29b1b1ba230f,38845,1732714839599, state=OPEN 2024-11-27T13:40:47,233 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44709-0x1017c8633370003, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:47,233 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35019-0x1017c8633370000, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:47,233 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1017c8633370001, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:47,233 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:47,233 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:47,233 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:47,234 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732714877234 2024-11-27T13:40:47,234 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T13:40:47,234 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-27T13:40:47,236 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-27T13:40:47,237 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-27T13:40:47,237 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true 2024-11-27T13:40:47,237 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T13:40:47,237 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1604) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-27T13:40:47,239 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51060408ec4ea38afe378df89549aeae, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,239 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b6b183b9bc56b2bc4dc6f8968ee01c8, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,239 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0aef6a7e4e138b1b9d7f4aea6971acc6, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,239 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49523b5c318014c3e33791ea6f5e308d, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,239 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4d1ad27263331aa341edfa82a041de56, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,239 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fc6390c5b3b9356bc4885792f770cf2, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,239 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85dc78011fc464d1446075ad14a217de, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,239 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ee34ecbedcb7dfdabcb99a67ac21045, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,239 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dcf91d39846df82f849110c861f2a7d0, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,240 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7a7c9c02e40375cd974ffe51b6da75b, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,240 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eeadb5717c90d7f7a33c12d4883bf2b0, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,240 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a9a679b7eb3754995869ddbc90163b1c, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,240 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cccf7ce6a567350b0ae162454b0a9379, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,240 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d095bb01efaabc5bf5541d373d5419fa, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,240 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c46624524e53a899138ac5a715a35d6, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,240 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de4deb0c16ab89c9c56fc8912227f1c5, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,240 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b448045aa14a354ef4bc74b08978138, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,240 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8700ed52628a2e71e47ec460999f2612, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,240 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5f0e25bfe9d22339132aab7b2f16f09, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,240 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=922622f55d3d471d091912fe46d7c8d8, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,240 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=457728e6ba667138202e8a0dfa04b5b7, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,240 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4f747499d4a9438c3cddbfb1ead0d2b, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,241 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed774066130adf09502c61d07d72f5ab, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,241 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c6bad09b1898b3a3810c7bfeb938da4, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,241 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d64103cc558d5243fdfd1e469558a599, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,241 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d41565a29ddd5c9153a0f41aba598ca, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,241 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-27T13:40:47,241 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732714907241 2024-11-27T13:40:47,241 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732714967241 2024-11-27T13:40:47,241 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-27T13:40:47,241 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-27T13:40:47,241 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T13:40:47,241 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-27T13:40:47,241 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T13:40:47,241 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-27T13:40:47,241 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-27T13:40:47,241 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-27T13:40:47,242 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T13:40:47,242 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-27T13:40:47,244 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-27T13:40:47,244 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T13:40:47,244 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-27T13:40:47,245 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true 2024-11-27T13:40:47,245 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-27T13:40:47,245 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-27T13:40:47,245 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=61 2024-11-27T13:40:47,246 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=29b1b1ba230f,38845,1732714839599, table=hbase:meta, region=1588230740 2024-11-27T13:40:47,246 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 29b1b1ba230f,38845,1732714839599, state=OPEN 2024-11-27T13:40:47,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35019-0x1017c8633370000, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:47,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1017c8633370001, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:47,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44709-0x1017c8633370003, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T13:40:47,257 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:47,257 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:47,257 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T13:40:47,258 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732714877258 2024-11-27T13:40:47,258 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T13:40:47,258 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-27T13:40:47,260 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-27T13:40:47,262 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=57, state=ROLLEDBACK, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure de4deb0c16ab89c9c56fc8912227f1c5, server=29b1b1ba230f,38845,1732714839599:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 29b1b1ba230f,39411,1732714839711, splitWal=true, meta=true exec-time=2.4850 sec 2024-11-27T13:40:47,263 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51060408ec4ea38afe378df89549aeae, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,263 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b6b183b9bc56b2bc4dc6f8968ee01c8, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,263 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0aef6a7e4e138b1b9d7f4aea6971acc6, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,263 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=49523b5c318014c3e33791ea6f5e308d, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,264 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4d1ad27263331aa341edfa82a041de56, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,264 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fc6390c5b3b9356bc4885792f770cf2, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,264 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85dc78011fc464d1446075ad14a217de, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,264 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ee34ecbedcb7dfdabcb99a67ac21045, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,264 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dcf91d39846df82f849110c861f2a7d0, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,264 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e7a7c9c02e40375cd974ffe51b6da75b, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,264 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eeadb5717c90d7f7a33c12d4883bf2b0, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,264 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a9a679b7eb3754995869ddbc90163b1c, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,264 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cccf7ce6a567350b0ae162454b0a9379, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,264 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d095bb01efaabc5bf5541d373d5419fa, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,264 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c46624524e53a899138ac5a715a35d6, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,264 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de4deb0c16ab89c9c56fc8912227f1c5, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,264 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b448045aa14a354ef4bc74b08978138, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,264 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8700ed52628a2e71e47ec460999f2612, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,265 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5f0e25bfe9d22339132aab7b2f16f09, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,265 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=922622f55d3d471d091912fe46d7c8d8, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,265 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=457728e6ba667138202e8a0dfa04b5b7, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,265 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4f747499d4a9438c3cddbfb1ead0d2b, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,265 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed774066130adf09502c61d07d72f5ab, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,265 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c6bad09b1898b3a3810c7bfeb938da4, regionState=OPEN, lastHost=29b1b1ba230f,44709,1732714839765, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,265 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d64103cc558d5243fdfd1e469558a599, regionState=OPENING, lastHost=29b1b1ba230f,39411,1732714839711, regionLocation=29b1b1ba230f,44709,1732714839765, openSeqNum=2 2024-11-27T13:40:47,265 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d41565a29ddd5c9153a0f41aba598ca, regionState=OPEN, lastHost=29b1b1ba230f,38845,1732714839599, regionLocation=29b1b1ba230f,38845,1732714839599, openSeqNum=2 2024-11-27T13:40:47,265 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-27T13:40:47,265 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732714907265 2024-11-27T13:40:47,265 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732714967265 2024-11-27T13:40:47,265 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-11-27T13:40:47,280 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.assignment.TestRollbackSCP#testFailAndRollback Thread=350 (was 385), OpenFileDescriptor=620 (was 643), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=259 (was 259), ProcessCount=11 (was 11), AvailableMemoryMB=9380 (was 9418) 2024-11-27T13:40:47,280 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-27T13:40:47,280 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T13:40:47,280 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T13:40:47,280 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T13:40:47,281 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T13:40:47,281 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T13:40:47,281 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-27T13:40:47,281 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=385927850, stopped=false 2024-11-27T13:40:47,281 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=29b1b1ba230f,35019,1732714838827 2024-11-27T13:40:47,289 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1017c8633370001, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-27T13:40:47,289 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44709-0x1017c8633370003, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-27T13:40:47,289 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35019-0x1017c8633370000, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-27T13:40:47,289 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44709-0x1017c8633370003, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T13:40:47,289 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35019-0x1017c8633370000, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T13:40:47,289 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T13:40:47,289 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1017c8633370001, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T13:40:47,289 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T13:40:47,289 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T13:40:47,290 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T13:40:47,290 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35019-0x1017c8633370000, quorum=127.0.0.1:60303, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T13:40:47,290 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44709-0x1017c8633370003, quorum=127.0.0.1:60303, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T13:40:47,290 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '29b1b1ba230f,38845,1732714839599' ***** 2024-11-27T13:40:47,290 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38845-0x1017c8633370001, quorum=127.0.0.1:60303, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T13:40:47,290 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-27T13:40:47,290 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '29b1b1ba230f,44709,1732714839765' ***** 2024-11-27T13:40:47,290 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-27T13:40:47,290 INFO [RS:0;29b1b1ba230f:38845 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-27T13:40:47,290 INFO [RS:2;29b1b1ba230f:44709 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-27T13:40:47,291 INFO [RS:0;29b1b1ba230f:38845 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-27T13:40:47,291 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-27T13:40:47,291 INFO [RS:2;29b1b1ba230f:44709 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-27T13:40:47,291 INFO [RS:0;29b1b1ba230f:38845 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-27T13:40:47,291 INFO [RS:2;29b1b1ba230f:44709 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-27T13:40:47,291 INFO [RS:2;29b1b1ba230f:44709 {}] regionserver.HRegionServer(3091): Received CLOSE for 8700ed52628a2e71e47ec460999f2612 2024-11-27T13:40:47,291 INFO [RS:0;29b1b1ba230f:38845 {}] regionserver.HRegionServer(3091): Received CLOSE for c4f747499d4a9438c3cddbfb1ead0d2b 2024-11-27T13:40:47,291 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-27T13:40:47,291 INFO [RS:2;29b1b1ba230f:44709 {}] regionserver.HRegionServer(3091): Received CLOSE for dcf91d39846df82f849110c861f2a7d0 2024-11-27T13:40:47,291 INFO [RS:2;29b1b1ba230f:44709 {}] regionserver.HRegionServer(3091): Received CLOSE for 5c6bad09b1898b3a3810c7bfeb938da4 2024-11-27T13:40:47,291 INFO [RS:2;29b1b1ba230f:44709 {}] regionserver.HRegionServer(3091): Received CLOSE for 49523b5c318014c3e33791ea6f5e308d 2024-11-27T13:40:47,291 INFO [RS:2;29b1b1ba230f:44709 {}] regionserver.HRegionServer(3091): Received CLOSE for 8fc6390c5b3b9356bc4885792f770cf2 2024-11-27T13:40:47,292 INFO [RS:2;29b1b1ba230f:44709 {}] regionserver.HRegionServer(3091): Received CLOSE for 0aef6a7e4e138b1b9d7f4aea6971acc6 2024-11-27T13:40:47,292 INFO [RS:2;29b1b1ba230f:44709 {}] regionserver.HRegionServer(3091): Received CLOSE for a9a679b7eb3754995869ddbc90163b1c 2024-11-27T13:40:47,292 INFO [RS:2;29b1b1ba230f:44709 {}] regionserver.HRegionServer(3091): Received CLOSE for 457728e6ba667138202e8a0dfa04b5b7 2024-11-27T13:40:47,292 INFO [RS:2;29b1b1ba230f:44709 {}] regionserver.HRegionServer(3091): Received CLOSE for 9c46624524e53a899138ac5a715a35d6 2024-11-27T13:40:47,292 INFO [RS:2;29b1b1ba230f:44709 {}] regionserver.HRegionServer(959): stopping server 29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:47,292 INFO [RS:2;29b1b1ba230f:44709 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-27T13:40:47,292 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8700ed52628a2e71e47ec460999f2612, disabling compactions & flushes 2024-11-27T13:40:47,292 INFO [RS:2;29b1b1ba230f:44709 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;29b1b1ba230f:44709. 2024-11-27T13:40:47,292 INFO [RS:0;29b1b1ba230f:38845 {}] regionserver.HRegionServer(3091): Received CLOSE for 6b448045aa14a354ef4bc74b08978138 2024-11-27T13:40:47,292 INFO [RS:0;29b1b1ba230f:38845 {}] regionserver.HRegionServer(3091): Received CLOSE for 51060408ec4ea38afe378df89549aeae 2024-11-27T13:40:47,292 DEBUG [RS:2;29b1b1ba230f:44709 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T13:40:47,292 INFO [RS:0;29b1b1ba230f:38845 {}] regionserver.HRegionServer(3091): Received CLOSE for 6d41565a29ddd5c9153a0f41aba598ca 2024-11-27T13:40:47,292 DEBUG [RS:2;29b1b1ba230f:44709 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T13:40:47,292 INFO [RS:0;29b1b1ba230f:38845 {}] regionserver.HRegionServer(3091): Received CLOSE for eeadb5717c90d7f7a33c12d4883bf2b0 2024-11-27T13:40:47,292 INFO [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,rrr,1732714842283.8700ed52628a2e71e47ec460999f2612. 2024-11-27T13:40:47,292 INFO [RS:0;29b1b1ba230f:38845 {}] regionserver.HRegionServer(3091): Received CLOSE for 85dc78011fc464d1446075ad14a217de 2024-11-27T13:40:47,292 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1732714842283.8700ed52628a2e71e47ec460999f2612. 2024-11-27T13:40:47,292 INFO [RS:0;29b1b1ba230f:38845 {}] regionserver.HRegionServer(3091): Received CLOSE for 4d1ad27263331aa341edfa82a041de56 2024-11-27T13:40:47,292 INFO [RS:0;29b1b1ba230f:38845 {}] regionserver.HRegionServer(3091): Received CLOSE for 2b6b183b9bc56b2bc4dc6f8968ee01c8 2024-11-27T13:40:47,292 INFO [RS:2;29b1b1ba230f:44709 {}] regionserver.HRegionServer(1321): Waiting on 9 regions to close 2024-11-27T13:40:47,292 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1732714842283.8700ed52628a2e71e47ec460999f2612. after waiting 0 ms 2024-11-27T13:40:47,292 INFO [RS:0;29b1b1ba230f:38845 {}] regionserver.HRegionServer(3091): Received CLOSE for e7a7c9c02e40375cd974ffe51b6da75b 2024-11-27T13:40:47,292 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1732714842283.8700ed52628a2e71e47ec460999f2612. 2024-11-27T13:40:47,292 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c4f747499d4a9438c3cddbfb1ead0d2b, disabling compactions & flushes 2024-11-27T13:40:47,292 INFO [RS:0;29b1b1ba230f:38845 {}] regionserver.HRegionServer(3091): Received CLOSE for d095bb01efaabc5bf5541d373d5419fa 2024-11-27T13:40:47,292 DEBUG [RS:2;29b1b1ba230f:44709 {}] regionserver.HRegionServer(1325): Online Regions={8700ed52628a2e71e47ec460999f2612=test,rrr,1732714842283.8700ed52628a2e71e47ec460999f2612., dcf91d39846df82f849110c861f2a7d0=test,iii,1732714842283.dcf91d39846df82f849110c861f2a7d0., 5c6bad09b1898b3a3810c7bfeb938da4=test,xxx,1732714842283.5c6bad09b1898b3a3810c7bfeb938da4., 49523b5c318014c3e33791ea6f5e308d=test,ddd,1732714842283.49523b5c318014c3e33791ea6f5e308d., 8fc6390c5b3b9356bc4885792f770cf2=test,fff,1732714842283.8fc6390c5b3b9356bc4885792f770cf2., 0aef6a7e4e138b1b9d7f4aea6971acc6=test,ccc,1732714842283.0aef6a7e4e138b1b9d7f4aea6971acc6., a9a679b7eb3754995869ddbc90163b1c=test,lll,1732714842283.a9a679b7eb3754995869ddbc90163b1c., 457728e6ba667138202e8a0dfa04b5b7=test,uuu,1732714842283.457728e6ba667138202e8a0dfa04b5b7., 9c46624524e53a899138ac5a715a35d6=test,ooo,1732714842283.9c46624524e53a899138ac5a715a35d6.} 2024-11-27T13:40:47,293 INFO [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,vvv,1732714842283.c4f747499d4a9438c3cddbfb1ead0d2b. 2024-11-27T13:40:47,293 INFO [RS:0;29b1b1ba230f:38845 {}] regionserver.HRegionServer(3091): Received CLOSE for 922622f55d3d471d091912fe46d7c8d8 2024-11-27T13:40:47,293 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1732714842283.c4f747499d4a9438c3cddbfb1ead0d2b. 2024-11-27T13:40:47,293 INFO [RS:0;29b1b1ba230f:38845 {}] regionserver.HRegionServer(3091): Received CLOSE for ed774066130adf09502c61d07d72f5ab 2024-11-27T13:40:47,293 INFO [RS:0;29b1b1ba230f:38845 {}] regionserver.HRegionServer(3091): Received CLOSE for de4deb0c16ab89c9c56fc8912227f1c5 2024-11-27T13:40:47,293 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1732714842283.c4f747499d4a9438c3cddbfb1ead0d2b. after waiting 0 ms 2024-11-27T13:40:47,293 INFO [RS:0;29b1b1ba230f:38845 {}] regionserver.HRegionServer(959): stopping server 29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:47,293 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1732714842283.c4f747499d4a9438c3cddbfb1ead0d2b. 2024-11-27T13:40:47,293 INFO [RS:0;29b1b1ba230f:38845 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-27T13:40:47,293 INFO [RS:0;29b1b1ba230f:38845 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;29b1b1ba230f:38845. 2024-11-27T13:40:47,293 DEBUG [RS:0;29b1b1ba230f:38845 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T13:40:47,293 DEBUG [RS:0;29b1b1ba230f:38845 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T13:40:47,293 DEBUG [RS:2;29b1b1ba230f:44709 {}] regionserver.HRegionServer(1351): Waiting on 0aef6a7e4e138b1b9d7f4aea6971acc6, 457728e6ba667138202e8a0dfa04b5b7, 49523b5c318014c3e33791ea6f5e308d, 5c6bad09b1898b3a3810c7bfeb938da4, 8700ed52628a2e71e47ec460999f2612, 8fc6390c5b3b9356bc4885792f770cf2, 9c46624524e53a899138ac5a715a35d6, a9a679b7eb3754995869ddbc90163b1c, d64103cc558d5243fdfd1e469558a599, dcf91d39846df82f849110c861f2a7d0 2024-11-27T13:40:47,293 INFO [RS:0;29b1b1ba230f:38845 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-27T13:40:47,293 INFO [RS:0;29b1b1ba230f:38845 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-27T13:40:47,293 INFO [RS:0;29b1b1ba230f:38845 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-27T13:40:47,293 INFO [RS:0;29b1b1ba230f:38845 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-27T13:40:47,294 INFO [RS:0;29b1b1ba230f:38845 {}] regionserver.HRegionServer(1321): Waiting on 14 regions to close 2024-11-27T13:40:47,294 DEBUG [RS:0;29b1b1ba230f:38845 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 2b6b183b9bc56b2bc4dc6f8968ee01c8, 4d1ad27263331aa341edfa82a041de56, 51060408ec4ea38afe378df89549aeae, 6b448045aa14a354ef4bc74b08978138, 6d41565a29ddd5c9153a0f41aba598ca, 85dc78011fc464d1446075ad14a217de, 922622f55d3d471d091912fe46d7c8d8, c4f747499d4a9438c3cddbfb1ead0d2b, d095bb01efaabc5bf5541d373d5419fa, de4deb0c16ab89c9c56fc8912227f1c5, e7a7c9c02e40375cd974ffe51b6da75b, ed774066130adf09502c61d07d72f5ab, eeadb5717c90d7f7a33c12d4883bf2b0 2024-11-27T13:40:47,294 DEBUG [RS_CLOSE_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-27T13:40:47,294 INFO [RS_CLOSE_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-27T13:40:47,294 DEBUG [RS_CLOSE_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-27T13:40:47,294 DEBUG [RS_CLOSE_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-27T13:40:47,294 DEBUG [RS_CLOSE_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-27T13:40:47,295 INFO [RS_CLOSE_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.66 KB heapSize=5.44 KB 2024-11-27T13:40:47,298 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/c4f747499d4a9438c3cddbfb1ead0d2b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T13:40:47,298 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/8700ed52628a2e71e47ec460999f2612/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T13:40:47,301 INFO [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,vvv,1732714842283.c4f747499d4a9438c3cddbfb1ead0d2b. 2024-11-27T13:40:47,301 INFO [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,rrr,1732714842283.8700ed52628a2e71e47ec460999f2612. 2024-11-27T13:40:47,301 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c4f747499d4a9438c3cddbfb1ead0d2b: Waiting for close lock at 1732714847292Running coprocessor pre-close hooks at 1732714847292Disabling compacts and flushes for region at 1732714847292Disabling writes for close at 1732714847293 (+1 ms)Writing region close event to WAL at 1732714847294 (+1 ms)Running coprocessor post-close hooks at 1732714847299 (+5 ms)Closed at 1732714847301 (+2 ms) 2024-11-27T13:40:47,301 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8700ed52628a2e71e47ec460999f2612: Waiting for close lock at 1732714847291Running coprocessor pre-close hooks at 1732714847292 (+1 ms)Disabling compacts and flushes for region at 1732714847292Disabling writes for close at 1732714847292Writing region close event to WAL at 1732714847293 (+1 ms)Running coprocessor post-close hooks at 1732714847299 (+6 ms)Closed at 1732714847301 (+2 ms) 2024-11-27T13:40:47,301 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,vvv,1732714842283.c4f747499d4a9438c3cddbfb1ead0d2b. 2024-11-27T13:40:47,301 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,rrr,1732714842283.8700ed52628a2e71e47ec460999f2612. 2024-11-27T13:40:47,302 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6b448045aa14a354ef4bc74b08978138, disabling compactions & flushes 2024-11-27T13:40:47,302 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing dcf91d39846df82f849110c861f2a7d0, disabling compactions & flushes 2024-11-27T13:40:47,302 INFO [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,qqq,1732714842283.6b448045aa14a354ef4bc74b08978138. 2024-11-27T13:40:47,302 INFO [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,iii,1732714842283.dcf91d39846df82f849110c861f2a7d0. 2024-11-27T13:40:47,302 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1732714842283.6b448045aa14a354ef4bc74b08978138. 2024-11-27T13:40:47,302 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1732714842283.dcf91d39846df82f849110c861f2a7d0. 2024-11-27T13:40:47,302 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1732714842283.6b448045aa14a354ef4bc74b08978138. after waiting 0 ms 2024-11-27T13:40:47,302 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,iii,1732714842283.dcf91d39846df82f849110c861f2a7d0. after waiting 0 ms 2024-11-27T13:40:47,302 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1732714842283.6b448045aa14a354ef4bc74b08978138. 2024-11-27T13:40:47,302 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,iii,1732714842283.dcf91d39846df82f849110c861f2a7d0. 2024-11-27T13:40:47,307 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/6b448045aa14a354ef4bc74b08978138/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T13:40:47,307 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/dcf91d39846df82f849110c861f2a7d0/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T13:40:47,308 INFO [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,iii,1732714842283.dcf91d39846df82f849110c861f2a7d0. 2024-11-27T13:40:47,308 INFO [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,qqq,1732714842283.6b448045aa14a354ef4bc74b08978138. 2024-11-27T13:40:47,308 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for dcf91d39846df82f849110c861f2a7d0: Waiting for close lock at 1732714847302Running coprocessor pre-close hooks at 1732714847302Disabling compacts and flushes for region at 1732714847302Disabling writes for close at 1732714847302Writing region close event to WAL at 1732714847302Running coprocessor post-close hooks at 1732714847308 (+6 ms)Closed at 1732714847308 2024-11-27T13:40:47,308 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6b448045aa14a354ef4bc74b08978138: Waiting for close lock at 1732714847302Running coprocessor pre-close hooks at 1732714847302Disabling compacts and flushes for region at 1732714847302Disabling writes for close at 1732714847302Writing region close event to WAL at 1732714847302Running coprocessor post-close hooks at 1732714847308 (+6 ms)Closed at 1732714847308 2024-11-27T13:40:47,308 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,iii,1732714842283.dcf91d39846df82f849110c861f2a7d0. 2024-11-27T13:40:47,308 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,qqq,1732714842283.6b448045aa14a354ef4bc74b08978138. 2024-11-27T13:40:47,309 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5c6bad09b1898b3a3810c7bfeb938da4, disabling compactions & flushes 2024-11-27T13:40:47,309 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 51060408ec4ea38afe378df89549aeae, disabling compactions & flushes 2024-11-27T13:40:47,309 INFO [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,xxx,1732714842283.5c6bad09b1898b3a3810c7bfeb938da4. 2024-11-27T13:40:47,309 INFO [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,,1732714842283.51060408ec4ea38afe378df89549aeae. 2024-11-27T13:40:47,309 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1732714842283.5c6bad09b1898b3a3810c7bfeb938da4. 2024-11-27T13:40:47,309 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1732714842283.51060408ec4ea38afe378df89549aeae. 2024-11-27T13:40:47,309 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1732714842283.5c6bad09b1898b3a3810c7bfeb938da4. after waiting 0 ms 2024-11-27T13:40:47,309 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1732714842283.5c6bad09b1898b3a3810c7bfeb938da4. 2024-11-27T13:40:47,309 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,,1732714842283.51060408ec4ea38afe378df89549aeae. after waiting 0 ms 2024-11-27T13:40:47,309 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,,1732714842283.51060408ec4ea38afe378df89549aeae. 2024-11-27T13:40:47,310 DEBUG [RS_CLOSE_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/hbase/meta/1588230740/.tmp/info/22f7d799eba54c6da0e7b1e2da89e003 is 138, key is test,bbb,1732714842283.2b6b183b9bc56b2bc4dc6f8968ee01c8./info:regioninfo/1732714846218/Put/seqid=0 2024-11-27T13:40:47,317 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/51060408ec4ea38afe378df89549aeae/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T13:40:47,317 INFO [regionserver/29b1b1ba230f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T13:40:47,317 INFO [regionserver/29b1b1ba230f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T13:40:47,318 INFO [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,,1732714842283.51060408ec4ea38afe378df89549aeae. 2024-11-27T13:40:47,318 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 51060408ec4ea38afe378df89549aeae: Waiting for close lock at 1732714847309Running coprocessor pre-close hooks at 1732714847309Disabling compacts and flushes for region at 1732714847309Disabling writes for close at 1732714847309Writing region close event to WAL at 1732714847312 (+3 ms)Running coprocessor post-close hooks at 1732714847318 (+6 ms)Closed at 1732714847318 2024-11-27T13:40:47,318 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,,1732714842283.51060408ec4ea38afe378df89549aeae. 2024-11-27T13:40:47,318 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/5c6bad09b1898b3a3810c7bfeb938da4/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T13:40:47,318 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6d41565a29ddd5c9153a0f41aba598ca, disabling compactions & flushes 2024-11-27T13:40:47,318 INFO [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,zzz,1732714842283.6d41565a29ddd5c9153a0f41aba598ca. 2024-11-27T13:40:47,318 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1732714842283.6d41565a29ddd5c9153a0f41aba598ca. 2024-11-27T13:40:47,318 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1732714842283.6d41565a29ddd5c9153a0f41aba598ca. after waiting 0 ms 2024-11-27T13:40:47,318 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1732714842283.6d41565a29ddd5c9153a0f41aba598ca. 2024-11-27T13:40:47,319 INFO [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,xxx,1732714842283.5c6bad09b1898b3a3810c7bfeb938da4. 2024-11-27T13:40:47,319 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5c6bad09b1898b3a3810c7bfeb938da4: Waiting for close lock at 1732714847309Running coprocessor pre-close hooks at 1732714847309Disabling compacts and flushes for region at 1732714847309Disabling writes for close at 1732714847309Writing region close event to WAL at 1732714847312 (+3 ms)Running coprocessor post-close hooks at 1732714847319 (+7 ms)Closed at 1732714847319 2024-11-27T13:40:47,319 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,xxx,1732714842283.5c6bad09b1898b3a3810c7bfeb938da4. 2024-11-27T13:40:47,319 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 49523b5c318014c3e33791ea6f5e308d, disabling compactions & flushes 2024-11-27T13:40:47,319 INFO [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ddd,1732714842283.49523b5c318014c3e33791ea6f5e308d. 2024-11-27T13:40:47,319 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1732714842283.49523b5c318014c3e33791ea6f5e308d. 2024-11-27T13:40:47,319 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1732714842283.49523b5c318014c3e33791ea6f5e308d. after waiting 0 ms 2024-11-27T13:40:47,319 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1732714842283.49523b5c318014c3e33791ea6f5e308d. 2024-11-27T13:40:47,329 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/6d41565a29ddd5c9153a0f41aba598ca/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T13:40:47,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42991 is added to blk_1073741869_1045 (size=8195) 2024-11-27T13:40:47,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37007 is added to blk_1073741869_1045 (size=8195) 2024-11-27T13:40:47,330 INFO [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,zzz,1732714842283.6d41565a29ddd5c9153a0f41aba598ca. 2024-11-27T13:40:47,330 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6d41565a29ddd5c9153a0f41aba598ca: Waiting for close lock at 1732714847318Running coprocessor pre-close hooks at 1732714847318Disabling compacts and flushes for region at 1732714847318Disabling writes for close at 1732714847318Writing region close event to WAL at 1732714847319 (+1 ms)Running coprocessor post-close hooks at 1732714847330 (+11 ms)Closed at 1732714847330 2024-11-27T13:40:47,331 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,zzz,1732714842283.6d41565a29ddd5c9153a0f41aba598ca. 2024-11-27T13:40:47,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38955 is added to blk_1073741869_1045 (size=8195) 2024-11-27T13:40:47,331 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing eeadb5717c90d7f7a33c12d4883bf2b0, disabling compactions & flushes 2024-11-27T13:40:47,331 INFO [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,kkk,1732714842283.eeadb5717c90d7f7a33c12d4883bf2b0. 2024-11-27T13:40:47,331 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1732714842283.eeadb5717c90d7f7a33c12d4883bf2b0. 2024-11-27T13:40:47,331 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1732714842283.eeadb5717c90d7f7a33c12d4883bf2b0. after waiting 0 ms 2024-11-27T13:40:47,331 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1732714842283.eeadb5717c90d7f7a33c12d4883bf2b0. 2024-11-27T13:40:47,332 INFO [RS_CLOSE_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.66 KB at sequenceid=71 (bloomFilter=true), to=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/hbase/meta/1588230740/.tmp/info/22f7d799eba54c6da0e7b1e2da89e003 2024-11-27T13:40:47,337 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/49523b5c318014c3e33791ea6f5e308d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T13:40:47,338 INFO [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ddd,1732714842283.49523b5c318014c3e33791ea6f5e308d. 2024-11-27T13:40:47,338 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 49523b5c318014c3e33791ea6f5e308d: Waiting for close lock at 1732714847319Running coprocessor pre-close hooks at 1732714847319Disabling compacts and flushes for region at 1732714847319Disabling writes for close at 1732714847319Writing region close event to WAL at 1732714847324 (+5 ms)Running coprocessor post-close hooks at 1732714847338 (+14 ms)Closed at 1732714847338 2024-11-27T13:40:47,338 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ddd,1732714842283.49523b5c318014c3e33791ea6f5e308d. 2024-11-27T13:40:47,338 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8fc6390c5b3b9356bc4885792f770cf2, disabling compactions & flushes 2024-11-27T13:40:47,338 INFO [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,fff,1732714842283.8fc6390c5b3b9356bc4885792f770cf2. 2024-11-27T13:40:47,338 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1732714842283.8fc6390c5b3b9356bc4885792f770cf2. 2024-11-27T13:40:47,338 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,fff,1732714842283.8fc6390c5b3b9356bc4885792f770cf2. after waiting 0 ms 2024-11-27T13:40:47,338 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,fff,1732714842283.8fc6390c5b3b9356bc4885792f770cf2. 2024-11-27T13:40:47,338 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/eeadb5717c90d7f7a33c12d4883bf2b0/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T13:40:47,339 INFO [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,kkk,1732714842283.eeadb5717c90d7f7a33c12d4883bf2b0. 2024-11-27T13:40:47,339 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for eeadb5717c90d7f7a33c12d4883bf2b0: Waiting for close lock at 1732714847331Running coprocessor pre-close hooks at 1732714847331Disabling compacts and flushes for region at 1732714847331Disabling writes for close at 1732714847331Writing region close event to WAL at 1732714847332 (+1 ms)Running coprocessor post-close hooks at 1732714847339 (+7 ms)Closed at 1732714847339 2024-11-27T13:40:47,340 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,kkk,1732714842283.eeadb5717c90d7f7a33c12d4883bf2b0. 2024-11-27T13:40:47,340 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 85dc78011fc464d1446075ad14a217de, disabling compactions & flushes 2024-11-27T13:40:47,340 INFO [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ggg,1732714842283.85dc78011fc464d1446075ad14a217de. 2024-11-27T13:40:47,340 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1732714842283.85dc78011fc464d1446075ad14a217de. 2024-11-27T13:40:47,340 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1732714842283.85dc78011fc464d1446075ad14a217de. after waiting 0 ms 2024-11-27T13:40:47,340 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1732714842283.85dc78011fc464d1446075ad14a217de. 2024-11-27T13:40:47,342 DEBUG [RS_CLOSE_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/hbase/meta/1588230740/.tmp/info/22f7d799eba54c6da0e7b1e2da89e003 as hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/hbase/meta/1588230740/info/22f7d799eba54c6da0e7b1e2da89e003 2024-11-27T13:40:47,343 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/8fc6390c5b3b9356bc4885792f770cf2/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T13:40:47,343 INFO [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,fff,1732714842283.8fc6390c5b3b9356bc4885792f770cf2. 2024-11-27T13:40:47,344 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8fc6390c5b3b9356bc4885792f770cf2: Waiting for close lock at 1732714847338Running coprocessor pre-close hooks at 1732714847338Disabling compacts and flushes for region at 1732714847338Disabling writes for close at 1732714847338Writing region close event to WAL at 1732714847339 (+1 ms)Running coprocessor post-close hooks at 1732714847343 (+4 ms)Closed at 1732714847343 2024-11-27T13:40:47,344 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,fff,1732714842283.8fc6390c5b3b9356bc4885792f770cf2. 2024-11-27T13:40:47,344 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0aef6a7e4e138b1b9d7f4aea6971acc6, disabling compactions & flushes 2024-11-27T13:40:47,344 INFO [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ccc,1732714842283.0aef6a7e4e138b1b9d7f4aea6971acc6. 2024-11-27T13:40:47,344 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1732714842283.0aef6a7e4e138b1b9d7f4aea6971acc6. 2024-11-27T13:40:47,344 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1732714842283.0aef6a7e4e138b1b9d7f4aea6971acc6. after waiting 0 ms 2024-11-27T13:40:47,344 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1732714842283.0aef6a7e4e138b1b9d7f4aea6971acc6. 2024-11-27T13:40:47,350 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/85dc78011fc464d1446075ad14a217de/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T13:40:47,351 INFO [RS_CLOSE_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/hbase/meta/1588230740/info/22f7d799eba54c6da0e7b1e2da89e003, entries=24, sequenceid=71, filesize=8.0 K 2024-11-27T13:40:47,351 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/0aef6a7e4e138b1b9d7f4aea6971acc6/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T13:40:47,351 INFO [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ggg,1732714842283.85dc78011fc464d1446075ad14a217de. 2024-11-27T13:40:47,352 INFO [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ccc,1732714842283.0aef6a7e4e138b1b9d7f4aea6971acc6. 2024-11-27T13:40:47,352 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 85dc78011fc464d1446075ad14a217de: Waiting for close lock at 1732714847340Running coprocessor pre-close hooks at 1732714847340Disabling compacts and flushes for region at 1732714847340Disabling writes for close at 1732714847340Writing region close event to WAL at 1732714847340Running coprocessor post-close hooks at 1732714847351 (+11 ms)Closed at 1732714847351 2024-11-27T13:40:47,352 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0aef6a7e4e138b1b9d7f4aea6971acc6: Waiting for close lock at 1732714847344Running coprocessor pre-close hooks at 1732714847344Disabling compacts and flushes for region at 1732714847344Disabling writes for close at 1732714847344Writing region close event to WAL at 1732714847344Running coprocessor post-close hooks at 1732714847352 (+8 ms)Closed at 1732714847352 2024-11-27T13:40:47,352 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ggg,1732714842283.85dc78011fc464d1446075ad14a217de. 2024-11-27T13:40:47,352 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ccc,1732714842283.0aef6a7e4e138b1b9d7f4aea6971acc6. 2024-11-27T13:40:47,352 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a9a679b7eb3754995869ddbc90163b1c, disabling compactions & flushes 2024-11-27T13:40:47,352 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4d1ad27263331aa341edfa82a041de56, disabling compactions & flushes 2024-11-27T13:40:47,352 INFO [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,lll,1732714842283.a9a679b7eb3754995869ddbc90163b1c. 2024-11-27T13:40:47,352 INFO [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,eee,1732714842283.4d1ad27263331aa341edfa82a041de56. 2024-11-27T13:40:47,352 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1732714842283.a9a679b7eb3754995869ddbc90163b1c. 2024-11-27T13:40:47,352 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1732714842283.4d1ad27263331aa341edfa82a041de56. 2024-11-27T13:40:47,352 INFO [RS_CLOSE_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.66 KB/2720, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 58ms, sequenceid=71, compaction requested=false 2024-11-27T13:40:47,352 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,eee,1732714842283.4d1ad27263331aa341edfa82a041de56. after waiting 0 ms 2024-11-27T13:40:47,352 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,lll,1732714842283.a9a679b7eb3754995869ddbc90163b1c. after waiting 0 ms 2024-11-27T13:40:47,352 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,eee,1732714842283.4d1ad27263331aa341edfa82a041de56. 2024-11-27T13:40:47,352 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,lll,1732714842283.a9a679b7eb3754995869ddbc90163b1c. 2024-11-27T13:40:47,358 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/a9a679b7eb3754995869ddbc90163b1c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T13:40:47,358 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/4d1ad27263331aa341edfa82a041de56/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T13:40:47,358 DEBUG [RS_CLOSE_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/hbase/meta/1588230740/recovered.edits/74.seqid, newMaxSeqId=74, maxSeqId=60 2024-11-27T13:40:47,359 INFO [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,eee,1732714842283.4d1ad27263331aa341edfa82a041de56. 2024-11-27T13:40:47,359 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4d1ad27263331aa341edfa82a041de56: Waiting for close lock at 1732714847352Running coprocessor pre-close hooks at 1732714847352Disabling compacts and flushes for region at 1732714847352Disabling writes for close at 1732714847352Writing region close event to WAL at 1732714847353 (+1 ms)Running coprocessor post-close hooks at 1732714847358 (+5 ms)Closed at 1732714847359 (+1 ms) 2024-11-27T13:40:47,359 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,eee,1732714842283.4d1ad27263331aa341edfa82a041de56. 2024-11-27T13:40:47,359 INFO [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,lll,1732714842283.a9a679b7eb3754995869ddbc90163b1c. 2024-11-27T13:40:47,359 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a9a679b7eb3754995869ddbc90163b1c: Waiting for close lock at 1732714847352Running coprocessor pre-close hooks at 1732714847352Disabling compacts and flushes for region at 1732714847352Disabling writes for close at 1732714847352Writing region close event to WAL at 1732714847353 (+1 ms)Running coprocessor post-close hooks at 1732714847359 (+6 ms)Closed at 1732714847359 2024-11-27T13:40:47,359 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2b6b183b9bc56b2bc4dc6f8968ee01c8, disabling compactions & flushes 2024-11-27T13:40:47,359 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,lll,1732714842283.a9a679b7eb3754995869ddbc90163b1c. 2024-11-27T13:40:47,359 INFO [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,bbb,1732714842283.2b6b183b9bc56b2bc4dc6f8968ee01c8. 2024-11-27T13:40:47,359 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1732714842283.2b6b183b9bc56b2bc4dc6f8968ee01c8. 2024-11-27T13:40:47,359 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 457728e6ba667138202e8a0dfa04b5b7, disabling compactions & flushes 2024-11-27T13:40:47,359 INFO [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,uuu,1732714842283.457728e6ba667138202e8a0dfa04b5b7. 2024-11-27T13:40:47,359 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1732714842283.2b6b183b9bc56b2bc4dc6f8968ee01c8. after waiting 0 ms 2024-11-27T13:40:47,359 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1732714842283.457728e6ba667138202e8a0dfa04b5b7. 2024-11-27T13:40:47,359 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1732714842283.2b6b183b9bc56b2bc4dc6f8968ee01c8. 2024-11-27T13:40:47,359 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1732714842283.457728e6ba667138202e8a0dfa04b5b7. after waiting 0 ms 2024-11-27T13:40:47,359 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1732714842283.457728e6ba667138202e8a0dfa04b5b7. 2024-11-27T13:40:47,360 DEBUG [RS_CLOSE_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-27T13:40:47,360 INFO [RS_CLOSE_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-27T13:40:47,360 DEBUG [RS_CLOSE_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732714847294Running coprocessor pre-close hooks at 1732714847294Disabling compacts and flushes for region at 1732714847294Disabling writes for close at 1732714847294Obtaining lock to block concurrent updates at 1732714847295 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1732714847295Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=2720, getHeapSize=5504, getOffHeapSize=0, getCellsCount=24 at 1732714847303 (+8 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732714847304 (+1 ms)Flushing 1588230740/info: creating writer at 1732714847304Flushing 1588230740/info: appending metadata at 1732714847310 (+6 ms)Flushing 1588230740/info: closing flushed file at 1732714847310Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4847c87e: reopening flushed file at 1732714847340 (+30 ms)Finished flush of dataSize ~2.66 KB/2720, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 58ms, sequenceid=71, compaction requested=false at 1732714847352 (+12 ms)Writing region close event to WAL at 1732714847355 (+3 ms)Running coprocessor post-close hooks at 1732714847360 (+5 ms)Closed at 1732714847360 2024-11-27T13:40:47,360 DEBUG [RS_CLOSE_META-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-27T13:40:47,364 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/457728e6ba667138202e8a0dfa04b5b7/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T13:40:47,364 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/2b6b183b9bc56b2bc4dc6f8968ee01c8/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T13:40:47,364 INFO [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,uuu,1732714842283.457728e6ba667138202e8a0dfa04b5b7. 2024-11-27T13:40:47,365 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 457728e6ba667138202e8a0dfa04b5b7: Waiting for close lock at 1732714847359Running coprocessor pre-close hooks at 1732714847359Disabling compacts and flushes for region at 1732714847359Disabling writes for close at 1732714847359Writing region close event to WAL at 1732714847360 (+1 ms)Running coprocessor post-close hooks at 1732714847364 (+4 ms)Closed at 1732714847364 2024-11-27T13:40:47,365 INFO [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,bbb,1732714842283.2b6b183b9bc56b2bc4dc6f8968ee01c8. 2024-11-27T13:40:47,365 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2b6b183b9bc56b2bc4dc6f8968ee01c8: Waiting for close lock at 1732714847359Running coprocessor pre-close hooks at 1732714847359Disabling compacts and flushes for region at 1732714847359Disabling writes for close at 1732714847359Writing region close event to WAL at 1732714847360 (+1 ms)Running coprocessor post-close hooks at 1732714847364 (+4 ms)Closed at 1732714847364 2024-11-27T13:40:47,365 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,uuu,1732714842283.457728e6ba667138202e8a0dfa04b5b7. 2024-11-27T13:40:47,365 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,bbb,1732714842283.2b6b183b9bc56b2bc4dc6f8968ee01c8. 2024-11-27T13:40:47,365 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9c46624524e53a899138ac5a715a35d6, disabling compactions & flushes 2024-11-27T13:40:47,365 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e7a7c9c02e40375cd974ffe51b6da75b, disabling compactions & flushes 2024-11-27T13:40:47,365 INFO [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ooo,1732714842283.9c46624524e53a899138ac5a715a35d6. 2024-11-27T13:40:47,365 INFO [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,jjj,1732714842283.e7a7c9c02e40375cd974ffe51b6da75b. 2024-11-27T13:40:47,365 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1732714842283.9c46624524e53a899138ac5a715a35d6. 2024-11-27T13:40:47,365 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1732714842283.e7a7c9c02e40375cd974ffe51b6da75b. 2024-11-27T13:40:47,365 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1732714842283.9c46624524e53a899138ac5a715a35d6. after waiting 0 ms 2024-11-27T13:40:47,365 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1732714842283.e7a7c9c02e40375cd974ffe51b6da75b. after waiting 0 ms 2024-11-27T13:40:47,365 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1732714842283.9c46624524e53a899138ac5a715a35d6. 2024-11-27T13:40:47,365 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1732714842283.e7a7c9c02e40375cd974ffe51b6da75b. 2024-11-27T13:40:47,369 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/e7a7c9c02e40375cd974ffe51b6da75b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T13:40:47,369 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/9c46624524e53a899138ac5a715a35d6/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T13:40:47,370 INFO [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,jjj,1732714842283.e7a7c9c02e40375cd974ffe51b6da75b. 2024-11-27T13:40:47,370 INFO [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ooo,1732714842283.9c46624524e53a899138ac5a715a35d6. 2024-11-27T13:40:47,370 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e7a7c9c02e40375cd974ffe51b6da75b: Waiting for close lock at 1732714847365Running coprocessor pre-close hooks at 1732714847365Disabling compacts and flushes for region at 1732714847365Disabling writes for close at 1732714847365Writing region close event to WAL at 1732714847365Running coprocessor post-close hooks at 1732714847370 (+5 ms)Closed at 1732714847370 2024-11-27T13:40:47,370 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9c46624524e53a899138ac5a715a35d6: Waiting for close lock at 1732714847365Running coprocessor pre-close hooks at 1732714847365Disabling compacts and flushes for region at 1732714847365Disabling writes for close at 1732714847365Writing region close event to WAL at 1732714847365Running coprocessor post-close hooks at 1732714847370 (+5 ms)Closed at 1732714847370 2024-11-27T13:40:47,370 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ooo,1732714842283.9c46624524e53a899138ac5a715a35d6. 2024-11-27T13:40:47,370 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,jjj,1732714842283.e7a7c9c02e40375cd974ffe51b6da75b. 2024-11-27T13:40:47,370 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d095bb01efaabc5bf5541d373d5419fa, disabling compactions & flushes 2024-11-27T13:40:47,370 INFO [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,nnn,1732714842283.d095bb01efaabc5bf5541d373d5419fa. 2024-11-27T13:40:47,371 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1732714842283.d095bb01efaabc5bf5541d373d5419fa. 2024-11-27T13:40:47,371 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1732714842283.d095bb01efaabc5bf5541d373d5419fa. after waiting 0 ms 2024-11-27T13:40:47,371 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1732714842283.d095bb01efaabc5bf5541d373d5419fa. 2024-11-27T13:40:47,374 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/d095bb01efaabc5bf5541d373d5419fa/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T13:40:47,375 INFO [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,nnn,1732714842283.d095bb01efaabc5bf5541d373d5419fa. 2024-11-27T13:40:47,375 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d095bb01efaabc5bf5541d373d5419fa: Waiting for close lock at 1732714847370Running coprocessor pre-close hooks at 1732714847370Disabling compacts and flushes for region at 1732714847370Disabling writes for close at 1732714847371 (+1 ms)Writing region close event to WAL at 1732714847371Running coprocessor post-close hooks at 1732714847375 (+4 ms)Closed at 1732714847375 2024-11-27T13:40:47,375 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,nnn,1732714842283.d095bb01efaabc5bf5541d373d5419fa. 2024-11-27T13:40:47,375 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 922622f55d3d471d091912fe46d7c8d8, disabling compactions & flushes 2024-11-27T13:40:47,375 INFO [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ttt,1732714842283.922622f55d3d471d091912fe46d7c8d8. 2024-11-27T13:40:47,375 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1732714842283.922622f55d3d471d091912fe46d7c8d8. 2024-11-27T13:40:47,375 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1732714842283.922622f55d3d471d091912fe46d7c8d8. after waiting 0 ms 2024-11-27T13:40:47,375 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1732714842283.922622f55d3d471d091912fe46d7c8d8. 2024-11-27T13:40:47,379 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/922622f55d3d471d091912fe46d7c8d8/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T13:40:47,380 INFO [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ttt,1732714842283.922622f55d3d471d091912fe46d7c8d8. 2024-11-27T13:40:47,380 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 922622f55d3d471d091912fe46d7c8d8: Waiting for close lock at 1732714847375Running coprocessor pre-close hooks at 1732714847375Disabling compacts and flushes for region at 1732714847375Disabling writes for close at 1732714847375Writing region close event to WAL at 1732714847376 (+1 ms)Running coprocessor post-close hooks at 1732714847380 (+4 ms)Closed at 1732714847380 2024-11-27T13:40:47,380 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ttt,1732714842283.922622f55d3d471d091912fe46d7c8d8. 2024-11-27T13:40:47,380 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ed774066130adf09502c61d07d72f5ab, disabling compactions & flushes 2024-11-27T13:40:47,380 INFO [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,www,1732714842283.ed774066130adf09502c61d07d72f5ab. 2024-11-27T13:40:47,380 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1732714842283.ed774066130adf09502c61d07d72f5ab. 2024-11-27T13:40:47,380 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,www,1732714842283.ed774066130adf09502c61d07d72f5ab. after waiting 0 ms 2024-11-27T13:40:47,380 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,www,1732714842283.ed774066130adf09502c61d07d72f5ab. 2024-11-27T13:40:47,384 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/ed774066130adf09502c61d07d72f5ab/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T13:40:47,385 INFO [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,www,1732714842283.ed774066130adf09502c61d07d72f5ab. 2024-11-27T13:40:47,385 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ed774066130adf09502c61d07d72f5ab: Waiting for close lock at 1732714847380Running coprocessor pre-close hooks at 1732714847380Disabling compacts and flushes for region at 1732714847380Disabling writes for close at 1732714847380Writing region close event to WAL at 1732714847381 (+1 ms)Running coprocessor post-close hooks at 1732714847385 (+4 ms)Closed at 1732714847385 2024-11-27T13:40:47,385 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,www,1732714842283.ed774066130adf09502c61d07d72f5ab. 2024-11-27T13:40:47,385 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing de4deb0c16ab89c9c56fc8912227f1c5, disabling compactions & flushes 2024-11-27T13:40:47,385 INFO [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ppp,1732714842283.de4deb0c16ab89c9c56fc8912227f1c5. 2024-11-27T13:40:47,385 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1732714842283.de4deb0c16ab89c9c56fc8912227f1c5. 2024-11-27T13:40:47,385 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1732714842283.de4deb0c16ab89c9c56fc8912227f1c5. after waiting 0 ms 2024-11-27T13:40:47,385 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1732714842283.de4deb0c16ab89c9c56fc8912227f1c5. 2024-11-27T13:40:47,389 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/data/default/test/de4deb0c16ab89c9c56fc8912227f1c5/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T13:40:47,390 INFO [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ppp,1732714842283.de4deb0c16ab89c9c56fc8912227f1c5. 2024-11-27T13:40:47,390 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for de4deb0c16ab89c9c56fc8912227f1c5: Waiting for close lock at 1732714847385Running coprocessor pre-close hooks at 1732714847385Disabling compacts and flushes for region at 1732714847385Disabling writes for close at 1732714847385Writing region close event to WAL at 1732714847386 (+1 ms)Running coprocessor post-close hooks at 1732714847390 (+4 ms)Closed at 1732714847390 2024-11-27T13:40:47,390 DEBUG [RS_CLOSE_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ppp,1732714842283.de4deb0c16ab89c9c56fc8912227f1c5. 2024-11-27T13:40:47,406 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T13:40:47,458 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test' 2024-11-27T13:40:47,493 INFO [RS:2;29b1b1ba230f:44709 {}] regionserver.HRegionServer(976): stopping server 29b1b1ba230f,44709,1732714839765; all regions closed. 2024-11-27T13:40:47,494 INFO [RS:0;29b1b1ba230f:38845 {}] regionserver.HRegionServer(976): stopping server 29b1b1ba230f,38845,1732714839599; all regions closed. 2024-11-27T13:40:47,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38955 is added to blk_1073741835_1011 (size=5194) 2024-11-27T13:40:47,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37007 is added to blk_1073741865_1041 (size=4204) 2024-11-27T13:40:47,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42991 is added to blk_1073741865_1041 (size=4204) 2024-11-27T13:40:47,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38955 is added to blk_1073741865_1041 (size=4204) 2024-11-27T13:40:47,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37007 is added to blk_1073741835_1011 (size=5194) 2024-11-27T13:40:47,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42991 is added to blk_1073741835_1011 (size=5194) 2024-11-27T13:40:47,501 DEBUG [RS:0;29b1b1ba230f:38845 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/oldWALs 2024-11-27T13:40:47,501 DEBUG [RS:2;29b1b1ba230f:44709 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/oldWALs 2024-11-27T13:40:47,501 INFO [RS:0;29b1b1ba230f:38845 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 29b1b1ba230f%2C38845%2C1732714839599.meta:.meta(num 1732714845453) 2024-11-27T13:40:47,501 INFO [RS:2;29b1b1ba230f:44709 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 29b1b1ba230f%2C44709%2C1732714839765:(num 1732714841273) 2024-11-27T13:40:47,501 DEBUG [RS:2;29b1b1ba230f:44709 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T13:40:47,502 INFO [RS:2;29b1b1ba230f:44709 {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T13:40:47,502 INFO [RS:2;29b1b1ba230f:44709 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-27T13:40:47,502 INFO [RS:2;29b1b1ba230f:44709 {}] hbase.ChoreService(370): Chore service for: regionserver/29b1b1ba230f:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-27T13:40:47,502 INFO [RS:2;29b1b1ba230f:44709 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-27T13:40:47,502 INFO [RS:2;29b1b1ba230f:44709 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-27T13:40:47,502 INFO [regionserver/29b1b1ba230f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-27T13:40:47,502 INFO [RS:2;29b1b1ba230f:44709 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-27T13:40:47,502 INFO [RS:2;29b1b1ba230f:44709 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-27T13:40:47,503 WARN [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] util.Threads(127): sleep interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.util.Threads.sleep(Threads.java:125) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2374) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T13:40:47,503 WARN [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,yyy,1732714842283.d64103cc558d5243fdfd1e469558a599., aborting... java.io.IOException: Failed to report opened region to master: test,yyy,1732714842283.d64103cc558d5243fdfd1e469558a599. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T13:40:47,503 ERROR [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2440): ***** ABORTING region server 29b1b1ba230f,44709,1732714839765: Failed to open region test,yyy,1732714842283.d64103cc558d5243fdfd1e469558a599. and can not recover ***** java.io.IOException: Failed to report opened region to master: test,yyy,1732714842283.d64103cc558d5243fdfd1e469558a599. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T13:40:47,503 ERROR [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-27T13:40:47,504 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-27T13:40:47,505 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-27T13:40:47,505 INFO [RS:2;29b1b1ba230f:44709 {}] executor.ExecutorService(108): Executor-9-RS_OPEN_REGION-regionserver/29b1b1ba230f:0 had [Event #36 of type M_RS_OPEN_REGION (class org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler), Event #37 of type M_RS_OPEN_REGION (class org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler), Event #38 of type M_RS_OPEN_REGION (class org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler)] on shutdown 2024-11-27T13:40:47,505 INFO [RS:2;29b1b1ba230f:44709 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44709 2024-11-27T13:40:47,508 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-27T13:40:47,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37007 is added to blk_1073741834_1010 (size=7064) 2024-11-27T13:40:47,508 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/WALs/29b1b1ba230f,38845,1732714839599/29b1b1ba230f%2C38845%2C1732714839599.1732714841273 not finished, retry = 0 2024-11-27T13:40:47,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38955 is added to blk_1073741834_1010 (size=7064) 2024-11-27T13:40:47,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42991 is added to blk_1073741834_1010 (size=7064) 2024-11-27T13:40:47,509 DEBUG [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-27T13:40:47,517 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44709-0x1017c8633370003, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/29b1b1ba230f,44709,1732714839765 2024-11-27T13:40:47,517 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35019-0x1017c8633370000, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-27T13:40:47,517 INFO [RS:2;29b1b1ba230f:44709 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-27T13:40:47,518 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [29b1b1ba230f,44709,1732714839765] 2024-11-27T13:40:47,520 INFO [RS_OPEN_REGION-regionserver/29b1b1ba230f:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 479150328 }, "NonHeapMemoryUsage": { "committed": 162856960, "init": 7667712, "max": -1, "used": 160262240 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "29b1b1ba230f", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 40645, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 132, "ProcessCallTime_min": 0, "ProcessCallTime_max": 28, "ProcessCallTime_mean": 2, "ProcessCallTime_25th_percentile": 7, "ProcessCallTime_median": 14, "ProcessCallTime_75th_percentile": 21, "ProcessCallTime_90th_percentile": 25, "ProcessCallTime_95th_percentile": 26, "ProcessCallTime_98th_percentile": 27, "ProcessCallTime_99th_percentile": 27, "ProcessCallTime_99.9th_percentile": 27, "ProcessCallTime_TimeRangeCount_0-1": 132, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 132, "QueueCallTime_min": 0, "QueueCallTime_max": 21, "QueueCallTime_mean": 1, "QueueCallTime_25th_percentile": 5, "QueueCallTime_median": 10, "QueueCallTime_75th_percentile": 15, "QueueCallTime_90th_percentile": 18, "QueueCallTime_95th_percentile": 19, "QueueCallTime_98th_percentile": 20, "QueueCallTime_99th_percentile": 20, "QueueCallTime_99.9th_percentile": 20, "QueueCallTime_TimeRangeCount_0-1": 132, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 132, "TotalCallTime_min": 0, "TotalCallTime_max": 28, "TotalCallTime_mean": 4, "TotalCallTime_25th_percentile": 7, "TotalCallTime_median": 14, "TotalCallTime_75th_percentile": 21, "TotalCallTime_90th_percentile": 25, "TotalCallTime_95th_percentile": 26, "TotalCallTime_98th_percentile": 27, "TotalCallTime_99th_percentile": 27, "TotalCallTime_99.9th_percentile": 27, "TotalCallTime_TimeRangeCount_0-1": 132, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 132, "ResponseSize_min": 0, "ResponseSize_max": 22855, "ResponseSize_mean": 4500, "ResponseSize_25th_percentile": 5713, "ResponseSize_median": 11427, "ResponseSize_75th_percentile": 17141, "ResponseSize_90th_percentile": 20569, "ResponseSize_95th_percentile": 21712, "ResponseSize_98th_percentile": 22397, "ResponseSize_99th_percentile": 22626, "ResponseSize_99.9th_percentile": 22832, "ResponseSize_SizeRangeCount_0-10": 132, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 132, "RequestSize_min": 81, "RequestSize_max": 8095, "RequestSize_mean": 279, "RequestSize_25th_percentile": 2084, "RequestSize_median": 4088, "RequestSize_75th_percentile": 6091, "RequestSize_90th_percentile": 7293, "RequestSize_95th_percentile": 7694, "RequestSize_98th_percentile": 7934, "RequestSize_99th_percentile": 8014, "RequestSize_99.9th_percentile": 8086, "RequestSize_SizeRangeCount_0-10": 132, "sentBytes": 459927 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "29b1b1ba230f", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:60303", "tag.serverName": "29b1b1ba230f,44709,1732714839765", "tag.clusterId": "80a0a673-acef-4b6a-956f-a08e628106dc", "tag.Context": "regionserver", "tag.Hostname": "29b1b1ba230f", "regionCount": 0, "storeCount": 9, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 2304, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1732714839765, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 38203, "localBytesRead": 38203, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 1.8, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 39391, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 9, "totalRowActionRequestCount": 9, "readRequestCount": 9, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 9, "rpcScanRequestCount": 9, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 9, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "ScanTime_TimeRangeCount_0-1": 9, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 9, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "ScanSize_SizeRangeCount_0-10": 9, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-27T13:40:47,538 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/29b1b1ba230f,44709,1732714839765 already deleted, retry=false 2024-11-27T13:40:47,538 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 29b1b1ba230f,44709,1732714839765 expired; onlineServers=1 2024-11-27T13:40:47,612 DEBUG [RS:0;29b1b1ba230f:38845 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/oldWALs 2024-11-27T13:40:47,612 INFO [RS:0;29b1b1ba230f:38845 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 29b1b1ba230f%2C38845%2C1732714839599:(num 1732714841273) 2024-11-27T13:40:47,612 DEBUG [RS:0;29b1b1ba230f:38845 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T13:40:47,612 INFO [RS:0;29b1b1ba230f:38845 {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T13:40:47,612 INFO [RS:0;29b1b1ba230f:38845 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-27T13:40:47,612 INFO [RS:0;29b1b1ba230f:38845 {}] hbase.ChoreService(370): Chore service for: regionserver/29b1b1ba230f:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-27T13:40:47,612 INFO [RS:0;29b1b1ba230f:38845 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-27T13:40:47,612 INFO [regionserver/29b1b1ba230f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-27T13:40:47,613 INFO [RS:0;29b1b1ba230f:38845 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38845 2024-11-27T13:40:47,626 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1017c8633370001, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/29b1b1ba230f,38845,1732714839599 2024-11-27T13:40:47,626 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35019-0x1017c8633370000, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-27T13:40:47,626 INFO [RS:0;29b1b1ba230f:38845 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-27T13:40:47,627 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [29b1b1ba230f,38845,1732714839599] 2024-11-27T13:40:47,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44709-0x1017c8633370003, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T13:40:47,628 INFO [RS:2;29b1b1ba230f:44709 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-27T13:40:47,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44709-0x1017c8633370003, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T13:40:47,628 INFO [RS:2;29b1b1ba230f:44709 {}] regionserver.HRegionServer(1031): Exiting; stopping=29b1b1ba230f,44709,1732714839765; zookeeper connection closed. 2024-11-27T13:40:47,628 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6172d1e5 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6172d1e5 2024-11-27T13:40:47,647 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/29b1b1ba230f,38845,1732714839599 already deleted, retry=false 2024-11-27T13:40:47,647 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 29b1b1ba230f,38845,1732714839599 expired; onlineServers=0 2024-11-27T13:40:47,647 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '29b1b1ba230f,35019,1732714838827' ***** 2024-11-27T13:40:47,647 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-27T13:40:47,648 INFO [M:0;29b1b1ba230f:35019 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-27T13:40:47,648 INFO [M:0;29b1b1ba230f:35019 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-27T13:40:47,648 DEBUG [M:0;29b1b1ba230f:35019 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-27T13:40:47,648 DEBUG [M:0;29b1b1ba230f:35019 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-27T13:40:47,648 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-27T13:40:47,648 DEBUG [master/29b1b1ba230f:0:becomeActiveMaster-HFileCleaner.small.0-1732714840879 {}] cleaner.HFileCleaner(306): Exit Thread[master/29b1b1ba230f:0:becomeActiveMaster-HFileCleaner.small.0-1732714840879,5,FailOnTimeoutGroup] 2024-11-27T13:40:47,648 DEBUG [master/29b1b1ba230f:0:becomeActiveMaster-HFileCleaner.large.0-1732714840878 {}] cleaner.HFileCleaner(306): Exit Thread[master/29b1b1ba230f:0:becomeActiveMaster-HFileCleaner.large.0-1732714840878,5,FailOnTimeoutGroup] 2024-11-27T13:40:47,648 INFO [M:0;29b1b1ba230f:35019 {}] hbase.ChoreService(370): Chore service for: master/29b1b1ba230f:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-27T13:40:47,649 INFO [M:0;29b1b1ba230f:35019 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-27T13:40:47,649 DEBUG [M:0;29b1b1ba230f:35019 {}] master.HMaster(1795): Stopping service threads 2024-11-27T13:40:47,649 INFO [M:0;29b1b1ba230f:35019 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-27T13:40:47,649 INFO [M:0;29b1b1ba230f:35019 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T13:40:47,649 INFO [M:0;29b1b1ba230f:35019 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-27T13:40:47,649 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-27T13:40:47,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35019-0x1017c8633370000, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-27T13:40:47,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35019-0x1017c8633370000, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T13:40:47,657 DEBUG [M:0;29b1b1ba230f:35019 {}] zookeeper.ZKUtil(347): master:35019-0x1017c8633370000, quorum=127.0.0.1:60303, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-27T13:40:47,658 WARN [M:0;29b1b1ba230f:35019 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-27T13:40:47,658 INFO [M:0;29b1b1ba230f:35019 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/.lastflushedseqids 2024-11-27T13:40:47,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37007 is added to blk_1073741870_1046 (size=1) 2024-11-27T13:40:47,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42991 is added to blk_1073741870_1046 (size=1) 2024-11-27T13:40:47,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38955 is added to blk_1073741870_1046 (size=1) 2024-11-27T13:40:47,671 INFO [M:0;29b1b1ba230f:35019 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T13:40:47,671 INFO [M:0;29b1b1ba230f:35019 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-27T13:40:47,671 DEBUG [M:0;29b1b1ba230f:35019 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-27T13:40:47,671 INFO [M:0;29b1b1ba230f:35019 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T13:40:47,671 DEBUG [M:0;29b1b1ba230f:35019 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T13:40:47,671 DEBUG [M:0;29b1b1ba230f:35019 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-27T13:40:47,671 DEBUG [M:0;29b1b1ba230f:35019 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T13:40:47,671 INFO [M:0;29b1b1ba230f:35019 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=248.78 KB heapSize=302.31 KB 2024-11-27T13:40:47,688 DEBUG [M:0;29b1b1ba230f:35019 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/30a4276c1d694b52a1960c17e8720165 is 82, key is hbase:meta,,1/info:regioninfo/1732714845771/Put/seqid=0 2024-11-27T13:40:47,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38955 is added to blk_1073741871_1047 (size=6071) 2024-11-27T13:40:47,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42991 is added to blk_1073741871_1047 (size=6071) 2024-11-27T13:40:47,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37007 is added to blk_1073741871_1047 (size=6071) 2024-11-27T13:40:47,697 INFO [M:0;29b1b1ba230f:35019 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=571 (bloomFilter=true), to=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/30a4276c1d694b52a1960c17e8720165 2024-11-27T13:40:47,722 DEBUG [M:0;29b1b1ba230f:35019 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8e95d1cbc465498d8379951cf14b3418 is 2990, key is \x00\x00\x00\x00\x00\x00\x009/proc:d/1732714847261/Put/seqid=0 2024-11-27T13:40:47,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38955 is added to blk_1073741872_1048 (size=12801) 2024-11-27T13:40:47,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42991 is added to blk_1073741872_1048 (size=12801) 2024-11-27T13:40:47,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37007 is added to blk_1073741872_1048 (size=12801) 2024-11-27T13:40:47,729 INFO [M:0;29b1b1ba230f:35019 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=247.50 KB at sequenceid=571 (bloomFilter=true), to=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8e95d1cbc465498d8379951cf14b3418 2024-11-27T13:40:47,737 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1017c8633370001, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T13:40:47,737 INFO [RS:0;29b1b1ba230f:38845 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-27T13:40:47,737 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38845-0x1017c8633370001, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T13:40:47,737 INFO [RS:0;29b1b1ba230f:38845 {}] regionserver.HRegionServer(1031): Exiting; stopping=29b1b1ba230f,38845,1732714839599; zookeeper connection closed. 2024-11-27T13:40:47,737 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1e9b99d3 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1e9b99d3 2024-11-27T13:40:47,737 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-27T13:40:47,750 DEBUG [M:0;29b1b1ba230f:35019 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d55c637e54194b3b89d700196f5341b5 is 69, key is 29b1b1ba230f,38845,1732714839599/rs:state/1732714840970/Put/seqid=0 2024-11-27T13:40:47,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37007 is added to blk_1073741873_1049 (size=5447) 2024-11-27T13:40:47,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42991 is added to blk_1073741873_1049 (size=5447) 2024-11-27T13:40:47,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38955 is added to blk_1073741873_1049 (size=5447) 2024-11-27T13:40:47,758 INFO [M:0;29b1b1ba230f:35019 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=571 (bloomFilter=true), to=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d55c637e54194b3b89d700196f5341b5 2024-11-27T13:40:47,765 INFO [M:0;29b1b1ba230f:35019 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d55c637e54194b3b89d700196f5341b5 2024-11-27T13:40:47,779 DEBUG [M:0;29b1b1ba230f:35019 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/a624586f456d4140829844b41e3a0baf is 52, key is load_balancer_on/state:d/1732714844614/Put/seqid=0 2024-11-27T13:40:47,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38955 is added to blk_1073741874_1050 (size=5058) 2024-11-27T13:40:47,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42991 is added to blk_1073741874_1050 (size=5058) 2024-11-27T13:40:47,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37007 is added to blk_1073741874_1050 (size=5058) 2024-11-27T13:40:47,787 INFO [M:0;29b1b1ba230f:35019 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=571 (bloomFilter=true), to=hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/a624586f456d4140829844b41e3a0baf 2024-11-27T13:40:47,795 DEBUG [M:0;29b1b1ba230f:35019 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/30a4276c1d694b52a1960c17e8720165 as hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/30a4276c1d694b52a1960c17e8720165 2024-11-27T13:40:47,803 INFO [M:0;29b1b1ba230f:35019 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/30a4276c1d694b52a1960c17e8720165, entries=14, sequenceid=571, filesize=5.9 K 2024-11-27T13:40:47,804 DEBUG [M:0;29b1b1ba230f:35019 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8e95d1cbc465498d8379951cf14b3418 as hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/8e95d1cbc465498d8379951cf14b3418 2024-11-27T13:40:47,810 INFO [M:0;29b1b1ba230f:35019 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/8e95d1cbc465498d8379951cf14b3418, entries=79, sequenceid=571, filesize=12.5 K 2024-11-27T13:40:47,812 DEBUG [M:0;29b1b1ba230f:35019 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d55c637e54194b3b89d700196f5341b5 as hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/d55c637e54194b3b89d700196f5341b5 2024-11-27T13:40:47,819 INFO [M:0;29b1b1ba230f:35019 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d55c637e54194b3b89d700196f5341b5 2024-11-27T13:40:47,819 INFO [M:0;29b1b1ba230f:35019 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/d55c637e54194b3b89d700196f5341b5, entries=3, sequenceid=571, filesize=5.3 K 2024-11-27T13:40:47,820 DEBUG [M:0;29b1b1ba230f:35019 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/a624586f456d4140829844b41e3a0baf as hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/a624586f456d4140829844b41e3a0baf 2024-11-27T13:40:47,826 INFO [M:0;29b1b1ba230f:35019 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43491/user/jenkins/test-data/9ce47c2c-67ea-d2f7-6efb-41a8e181b237/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/a624586f456d4140829844b41e3a0baf, entries=1, sequenceid=571, filesize=4.9 K 2024-11-27T13:40:47,827 INFO [M:0;29b1b1ba230f:35019 {}] regionserver.HRegion(3140): Finished flush of dataSize ~248.78 KB/254748, heapSize ~302.25 KB/309504, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 156ms, sequenceid=571, compaction requested=false 2024-11-27T13:40:47,828 INFO [M:0;29b1b1ba230f:35019 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T13:40:47,829 DEBUG [M:0;29b1b1ba230f:35019 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732714847671Disabling compacts and flushes for region at 1732714847671Disabling writes for close at 1732714847671Obtaining lock to block concurrent updates at 1732714847671Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732714847671Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=254748, getHeapSize=309504, getOffHeapSize=0, getCellsCount=710 at 1732714847672 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732714847672Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732714847672Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732714847688 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732714847688Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732714847704 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732714847721 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732714847721Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732714847734 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732714847749 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732714847749Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732714847765 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732714847779 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732714847779Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@52300e8a: reopening flushed file at 1732714847794 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3db76cb0: reopening flushed file at 1732714847803 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@225a81d8: reopening flushed file at 1732714847810 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1baf6f5e: reopening flushed file at 1732714847819 (+9 ms)Finished flush of dataSize ~248.78 KB/254748, heapSize ~302.25 KB/309504, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 156ms, sequenceid=571, compaction requested=false at 1732714847827 (+8 ms)Writing region close event to WAL at 1732714847828 (+1 ms)Closed at 1732714847828 2024-11-27T13:40:47,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37007 is added to blk_1073741830_1006 (size=292872) 2024-11-27T13:40:47,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38955 is added to blk_1073741830_1006 (size=292872) 2024-11-27T13:40:47,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42991 is added to blk_1073741830_1006 (size=292872) 2024-11-27T13:40:47,832 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-27T13:40:47,833 INFO [M:0;29b1b1ba230f:35019 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-27T13:40:47,833 INFO [M:0;29b1b1ba230f:35019 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35019 2024-11-27T13:40:47,833 INFO [M:0;29b1b1ba230f:35019 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-27T13:40:47,943 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35019-0x1017c8633370000, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T13:40:47,943 INFO [M:0;29b1b1ba230f:35019 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-27T13:40:47,943 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35019-0x1017c8633370000, quorum=127.0.0.1:60303, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T13:40:47,947 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3c1f701{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-27T13:40:47,949 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3d599bb4{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-27T13:40:47,949 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-27T13:40:47,949 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@417efacb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-27T13:40:47,949 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@290e2eec{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8706bd9e-726a-c9fe-5644-8f636122acec/hadoop.log.dir/,STOPPED} 2024-11-27T13:40:47,952 WARN [BP-1944179033-172.17.0.2-1732714834229 heartbeating to localhost/127.0.0.1:43491 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-27T13:40:47,952 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-27T13:40:47,952 WARN [BP-1944179033-172.17.0.2-1732714834229 heartbeating to localhost/127.0.0.1:43491 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1944179033-172.17.0.2-1732714834229 (Datanode Uuid ad7930af-671f-4232-96bc-4c380afba2bb) service to localhost/127.0.0.1:43491 2024-11-27T13:40:47,952 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-27T13:40:47,953 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8706bd9e-726a-c9fe-5644-8f636122acec/cluster_192d029c-90d2-1a83-1d8c-c62dd1a1bf62/data/data5/current/BP-1944179033-172.17.0.2-1732714834229 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-27T13:40:47,953 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8706bd9e-726a-c9fe-5644-8f636122acec/cluster_192d029c-90d2-1a83-1d8c-c62dd1a1bf62/data/data6/current/BP-1944179033-172.17.0.2-1732714834229 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-27T13:40:47,954 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-27T13:40:47,956 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@34fcb585{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-27T13:40:47,956 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@389bb6f7{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-27T13:40:47,956 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-27T13:40:47,956 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1b0a32ba{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-27T13:40:47,957 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@29652cf4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8706bd9e-726a-c9fe-5644-8f636122acec/hadoop.log.dir/,STOPPED} 2024-11-27T13:40:47,958 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-27T13:40:47,958 WARN [BP-1944179033-172.17.0.2-1732714834229 heartbeating to localhost/127.0.0.1:43491 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-27T13:40:47,958 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-27T13:40:47,958 WARN [BP-1944179033-172.17.0.2-1732714834229 heartbeating to localhost/127.0.0.1:43491 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1944179033-172.17.0.2-1732714834229 (Datanode Uuid 1e6b559b-6530-485d-8c8d-7b2b8f3a5bf6) service to localhost/127.0.0.1:43491 2024-11-27T13:40:47,958 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8706bd9e-726a-c9fe-5644-8f636122acec/cluster_192d029c-90d2-1a83-1d8c-c62dd1a1bf62/data/data3/current/BP-1944179033-172.17.0.2-1732714834229 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-27T13:40:47,959 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8706bd9e-726a-c9fe-5644-8f636122acec/cluster_192d029c-90d2-1a83-1d8c-c62dd1a1bf62/data/data4/current/BP-1944179033-172.17.0.2-1732714834229 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-27T13:40:47,959 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-27T13:40:47,961 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@79b30fb0{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-27T13:40:47,961 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@548cd725{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-27T13:40:47,961 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-27T13:40:47,961 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@474d518f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-27T13:40:47,961 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3a300759{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8706bd9e-726a-c9fe-5644-8f636122acec/hadoop.log.dir/,STOPPED} 2024-11-27T13:40:47,962 WARN [BP-1944179033-172.17.0.2-1732714834229 heartbeating to localhost/127.0.0.1:43491 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-27T13:40:47,962 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-27T13:40:47,962 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-27T13:40:47,962 WARN [BP-1944179033-172.17.0.2-1732714834229 heartbeating to localhost/127.0.0.1:43491 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1944179033-172.17.0.2-1732714834229 (Datanode Uuid fced76f7-d573-4c9e-914b-bbbe29602243) service to localhost/127.0.0.1:43491 2024-11-27T13:40:47,963 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8706bd9e-726a-c9fe-5644-8f636122acec/cluster_192d029c-90d2-1a83-1d8c-c62dd1a1bf62/data/data1/current/BP-1944179033-172.17.0.2-1732714834229 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-27T13:40:47,963 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8706bd9e-726a-c9fe-5644-8f636122acec/cluster_192d029c-90d2-1a83-1d8c-c62dd1a1bf62/data/data2/current/BP-1944179033-172.17.0.2-1732714834229 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-27T13:40:47,964 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-27T13:40:47,972 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@64418798{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-27T13:40:47,972 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@11744524{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-27T13:40:47,973 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-27T13:40:47,973 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@595320c8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-27T13:40:47,973 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3912242{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8706bd9e-726a-c9fe-5644-8f636122acec/hadoop.log.dir/,STOPPED} 2024-11-27T13:40:47,983 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-27T13:40:48,029 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down